1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4	*/
5
6	#include <string.h>
7	#include <stdlib.h>
8	#include <inttypes.h>
9	#include <sys/mman.h>
10
11	#include <objtool/builtin.h>
12	#include <objtool/cfi.h>
13	#include <objtool/arch.h>
14	#include <objtool/check.h>
15	#include <objtool/special.h>
16	#include <objtool/warn.h>
17	#include <objtool/endianness.h>
18
19	#include <linux/objtool_types.h>
20	#include <linux/hashtable.h>
21	#include <linux/kernel.h>
22	#include <linux/static_call_types.h>
23	#include <linux/string.h>
24
25	struct alternative {
26	struct alternative *next;
27	struct instruction *insn;
28	bool skip_orig;
29	};
30
31	static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32
33	static struct cfi_init_state initial_func_cfi;
34	static struct cfi_state init_cfi;
35	static struct cfi_state func_cfi;
36	static struct cfi_state force_undefined_cfi;
37
38	struct instruction *find_insn(struct objtool_file *file,
39	struct section *sec, unsigned long offset)
40	{
41	struct instruction *insn;
42
43	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
44	if (insn->sec == sec && insn->offset == offset)
45	return insn;
46	}
47
48	return NULL;
49	}
50
51	struct instruction *next_insn_same_sec(struct objtool_file *file,
52	struct instruction *insn)
53	{
54	if (insn->idx == INSN_CHUNK_MAX)
55	return find_insn(file, sec: insn->sec, offset: insn->offset + insn->len);
56
57	insn++;
58	if (!insn->len)
59	return NULL;
60
61	return insn;
62	}
63
64	static struct instruction *next_insn_same_func(struct objtool_file *file,
65	struct instruction *insn)
66	{
67	struct instruction *next = next_insn_same_sec(file, insn);
68	struct symbol *func = insn_func(insn);
69
70	if (!func)
71	return NULL;
72
73	if (next && insn_func(next) == func)
74	return next;
75
76	/* Check if we're already in the subfunction: */
77	if (func == func->cfunc)
78	return NULL;
79
80	/* Move to the subfunction: */
81	return find_insn(file, sec: func->cfunc->sec, offset: func->cfunc->offset);
82	}
83
84	static struct instruction *prev_insn_same_sec(struct objtool_file *file,
85	struct instruction *insn)
86	{
87	if (insn->idx == 0) {
88	if (insn->prev_len)
89	return find_insn(file, sec: insn->sec, offset: insn->offset - insn->prev_len);
90	return NULL;
91	}
92
93	return insn - 1;
94	}
95
96	static struct instruction *prev_insn_same_sym(struct objtool_file *file,
97	struct instruction *insn)
98	{
99	struct instruction *prev = prev_insn_same_sec(file, insn);
100
101	if (prev && insn_func(prev) == insn_func(insn))
102	return prev;
103
104	return NULL;
105	}
106
107	#define for_each_insn(file, insn) \
108	for (struct section *__sec, *__fake = (struct section *)1; \
109	__fake; __fake = NULL) \
110	for_each_sec(file, __sec) \
111	sec_for_each_insn(file, __sec, insn)
112
113	#define func_for_each_insn(file, func, insn) \
114	for (insn = find_insn(file, func->sec, func->offset); \
115	insn; \
116	insn = next_insn_same_func(file, insn))
117
118	#define sym_for_each_insn(file, sym, insn) \
119	for (insn = find_insn(file, sym->sec, sym->offset); \
120	insn && insn->offset < sym->offset + sym->len; \
121	insn = next_insn_same_sec(file, insn))
122
123	#define sym_for_each_insn_continue_reverse(file, sym, insn) \
124	for (insn = prev_insn_same_sec(file, insn); \
125	insn && insn->offset >= sym->offset; \
126	insn = prev_insn_same_sec(file, insn))
127
128	#define sec_for_each_insn_from(file, insn) \
129	for (; insn; insn = next_insn_same_sec(file, insn))
130
131	#define sec_for_each_insn_continue(file, insn) \
132	for (insn = next_insn_same_sec(file, insn); insn; \
133	insn = next_insn_same_sec(file, insn))
134
135	static inline struct symbol *insn_call_dest(struct instruction *insn)
136	{
137	if (insn->type == INSN_JUMP_DYNAMIC ||
138	insn->type == INSN_CALL_DYNAMIC)
139	return NULL;
140
141	return insn->_call_dest;
142	}
143
144	static inline struct reloc *insn_jump_table(struct instruction *insn)
145	{
146	if (insn->type == INSN_JUMP_DYNAMIC ||
147	insn->type == INSN_CALL_DYNAMIC)
148	return insn->_jump_table;
149
150	return NULL;
151	}
152
153	static bool is_jump_table_jump(struct instruction *insn)
154	{
155	struct alt_group *alt_group = insn->alt_group;
156
157	if (insn_jump_table(insn))
158	return true;
159
160	/* Retpoline alternative for a jump table? */
161	return alt_group && alt_group->orig_group &&
162	insn_jump_table(insn: alt_group->orig_group->first_insn);
163	}
164
165	static bool is_sibling_call(struct instruction *insn)
166	{
167	/*
168	* Assume only STT_FUNC calls have jump-tables.
169	*/
170	if (insn_func(insn)) {
171	/* An indirect jump is either a sibling call or a jump to a table. */
172	if (insn->type == INSN_JUMP_DYNAMIC)
173	return !is_jump_table_jump(insn);
174	}
175
176	/* add_jump_destinations() sets insn_call_dest(insn) for sibling calls. */
177	return (is_static_jump(insn) && insn_call_dest(insn));
178	}
179
180	/*
181	* This checks to see if the given function is a "noreturn" function.
182	*
183	* For global functions which are outside the scope of this object file, we
184	* have to keep a manual list of them.
185	*
186	* For local functions, we have to detect them manually by simply looking for
187	* the lack of a return instruction.
188	*/
189	static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
190	int recursion)
191	{
192	int i;
193	struct instruction *insn;
194	bool empty = true;
195
196	#define NORETURN(func) __stringify(func),
197	static const char * const global_noreturns[] = {
198	#include "noreturns.h"
199	};
200	#undef NORETURN
201
202	if (!func)
203	return false;
204
205	if (func->bind == STB_GLOBAL || func->bind == STB_WEAK)
206	for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
207	if (!strcmp(func->name, global_noreturns[i]))
208	return true;
209
210	if (func->bind == STB_WEAK)
211	return false;
212
213	if (!func->len)
214	return false;
215
216	insn = find_insn(file, sec: func->sec, offset: func->offset);
217	if (!insn || !insn_func(insn))
218	return false;
219
220	func_for_each_insn(file, func, insn) {
221	empty = false;
222
223	if (insn->type == INSN_RETURN)
224	return false;
225	}
226
227	if (empty)
228	return false;
229
230	/*
231	* A function can have a sibling call instead of a return. In that
232	* case, the function's dead-end status depends on whether the target
233	* of the sibling call returns.
234	*/
235	func_for_each_insn(file, func, insn) {
236	if (is_sibling_call(insn)) {
237	struct instruction *dest = insn->jump_dest;
238
239	if (!dest)
240	/* sibling call to another file */
241	return false;
242
243	/* local sibling call */
244	if (recursion == 5) {
245	/*
246	* Infinite recursion: two functions have
247	* sibling calls to each other. This is a very
248	* rare case. It means they aren't dead ends.
249	*/
250	return false;
251	}
252
253	return __dead_end_function(file, func: insn_func(dest), recursion: recursion+1);
254	}
255	}
256
257	return true;
258	}
259
260	static bool dead_end_function(struct objtool_file *file, struct symbol *func)
261	{
262	return __dead_end_function(file, func, recursion: 0);
263	}
264
265	static void init_cfi_state(struct cfi_state *cfi)
266	{
267	int i;
268
269	for (i = 0; i < CFI_NUM_REGS; i++) {
270	cfi->regs[i].base = CFI_UNDEFINED;
271	cfi->vals[i].base = CFI_UNDEFINED;
272	}
273	cfi->cfa.base = CFI_UNDEFINED;
274	cfi->drap_reg = CFI_UNDEFINED;
275	cfi->drap_offset = -1;
276	}
277
278	static void init_insn_state(struct objtool_file *file, struct insn_state *state,
279	struct section *sec)
280	{
281	memset(state, 0, sizeof(*state));
282	init_cfi_state(cfi: &state->cfi);
283
284	/*
285	* We need the full vmlinux for noinstr validation, otherwise we can
286	* not correctly determine insn_call_dest(insn)->sec (external symbols
287	* do not have a section).
288	*/
289	if (opts.link && opts.noinstr && sec)
290	state->noinstr = sec->noinstr;
291	}
292
293	static struct cfi_state *cfi_alloc(void)
294	{
295	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
296	if (!cfi) {
297	WARN("calloc failed");
298	exit(1);
299	}
300	nr_cfi++;
301	return cfi;
302	}
303
304	static int cfi_bits;
305	static struct hlist_head *cfi_hash;
306
307	static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
308	{
309	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
310	(void *)cfi2 + sizeof(cfi2->hash),
311	sizeof(struct cfi_state) - sizeof(struct hlist_node));
312	}
313
314	static inline u32 cfi_key(struct cfi_state *cfi)
315	{
316	return jhash((void *)cfi + sizeof(cfi->hash),
317	sizeof(*cfi) - sizeof(cfi->hash), 0);
318	}
319
320	static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
321	{
322	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
323	struct cfi_state *obj;
324
325	hlist_for_each_entry(obj, head, hash) {
326	if (!cficmp(cfi1: cfi, cfi2: obj)) {
327	nr_cfi_cache++;
328	return obj;
329	}
330	}
331
332	obj = cfi_alloc();
333	*obj = *cfi;
334	hlist_add_head(n: &obj->hash, h: head);
335
336	return obj;
337	}
338
339	static void cfi_hash_add(struct cfi_state *cfi)
340	{
341	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
342
343	hlist_add_head(n: &cfi->hash, h: head);
344	}
345
346	static void *cfi_hash_alloc(unsigned long size)
347	{
348	cfi_bits = max(10, ilog2(size));
349	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
350	PROT_READ|PROT_WRITE,
351	MAP_PRIVATE|MAP_ANON, -1, 0);
352	if (cfi_hash == (void *)-1L) {
353	WARN("mmap fail cfi_hash");
354	cfi_hash = NULL;
355	} else if (opts.stats) {
356	printf("cfi_bits: %d\n", cfi_bits);
357	}
358
359	return cfi_hash;
360	}
361
362	static unsigned long nr_insns;
363	static unsigned long nr_insns_visited;
364
365	/*
366	* Call the arch-specific instruction decoder for all the instructions and add
367	* them to the global instruction list.
368	*/
369	static int decode_instructions(struct objtool_file *file)
370	{
371	struct section *sec;
372	struct symbol *func;
373	unsigned long offset;
374	struct instruction *insn;
375	int ret;
376
377	for_each_sec(file, sec) {
378	struct instruction *insns = NULL;
379	u8 prev_len = 0;
380	u8 idx = 0;
381
382	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
383	continue;
384
385	if (strcmp(sec->name, ".altinstr_replacement") &&
386	strcmp(sec->name, ".altinstr_aux") &&
387	strncmp(sec->name, ".discard.", 9))
388	sec->text = true;
389
390	if (!strcmp(sec->name, ".noinstr.text") ||
391	!strcmp(sec->name, ".entry.text") ||
392	!strcmp(sec->name, ".cpuidle.text") ||
393	!strncmp(sec->name, ".text..__x86.", 13))
394	sec->noinstr = true;
395
396	/*
397	* .init.text code is ran before userspace and thus doesn't
398	* strictly need retpolines, except for modules which are
399	* loaded late, they very much do need retpoline in their
400	* .init.text
401	*/
402	if (!strcmp(sec->name, ".init.text") && !opts.module)
403	sec->init = true;
404
405	for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
406	if (!insns || idx == INSN_CHUNK_MAX) {
407	insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
408	if (!insns) {
409	WARN("malloc failed");
410	return -1;
411	}
412	idx = 0;
413	} else {
414	idx++;
415	}
416	insn = &insns[idx];
417	insn->idx = idx;
418
419	INIT_LIST_HEAD(list: &insn->call_node);
420	insn->sec = sec;
421	insn->offset = offset;
422	insn->prev_len = prev_len;
423
424	ret = arch_decode_instruction(file, sec, offset,
425	sec->sh.sh_size - offset,
426	insn);
427	if (ret)
428	return ret;
429
430	prev_len = insn->len;
431
432	/*
433	* By default, "ud2" is a dead end unless otherwise
434	* annotated, because GCC 7 inserts it for certain
435	* divide-by-zero cases.
436	*/
437	if (insn->type == INSN_BUG)
438	insn->dead_end = true;
439
440	hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
441	nr_insns++;
442	}
443
444	// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
445
446	sec_for_each_sym(sec, func) {
447	if (func->type != STT_NOTYPE && func->type != STT_FUNC)
448	continue;
449
450	if (func->offset == sec->sh.sh_size) {
451	/* Heuristic: likely an "end" symbol */
452	if (func->type == STT_NOTYPE)
453	continue;
454	WARN("%s(): STT_FUNC at end of section",
455	func->name);
456	return -1;
457	}
458
459	if (func->embedded_insn || func->alias != func)
460	continue;
461
462	if (!find_insn(file, sec, offset: func->offset)) {
463	WARN("%s(): can't find starting instruction",
464	func->name);
465	return -1;
466	}
467
468	sym_for_each_insn(file, func, insn) {
469	insn->sym = func;
470	if (func->type == STT_FUNC &&
471	insn->type == INSN_ENDBR &&
472	list_empty(head: &insn->call_node)) {
473	if (insn->offset == func->offset) {
474	list_add_tail(new: &insn->call_node, head: &file->endbr_list);
475	file->nr_endbr++;
476	} else {
477	file->nr_endbr_int++;
478	}
479	}
480	}
481	}
482	}
483
484	if (opts.stats)
485	printf("nr_insns: %lu\n", nr_insns);
486
487	return 0;
488	}
489
490	/*
491	* Read the pv_ops[] .data table to find the static initialized values.
492	*/
493	static int add_pv_ops(struct objtool_file *file, const char *symname)
494	{
495	struct symbol *sym, *func;
496	unsigned long off, end;
497	struct reloc *reloc;
498	int idx;
499
500	sym = find_symbol_by_name(file->elf, symname);
501	if (!sym)
502	return 0;
503
504	off = sym->offset;
505	end = off + sym->len;
506	for (;;) {
507	reloc = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
508	if (!reloc)
509	break;
510
511	func = reloc->sym;
512	if (func->type == STT_SECTION)
513	func = find_symbol_by_offset(reloc->sym->sec,
514	reloc_addend(reloc));
515
516	idx = (reloc_offset(reloc) - sym->offset) / sizeof(unsigned long);
517
518	objtool_pv_add(file, idx, func);
519
520	off = reloc_offset(reloc) + 1;
521	if (off > end)
522	break;
523	}
524
525	return 0;
526	}
527
528	/*
529	* Allocate and initialize file->pv_ops[].
530	*/
531	static int init_pv_ops(struct objtool_file *file)
532	{
533	static const char *pv_ops_tables[] = {
534	"pv_ops",
535	"xen_cpu_ops",
536	"xen_irq_ops",
537	"xen_mmu_ops",
538	NULL,
539	};
540	const char *pv_ops;
541	struct symbol *sym;
542	int idx, nr;
543
544	if (!opts.noinstr)
545	return 0;
546
547	file->pv_ops = NULL;
548
549	sym = find_symbol_by_name(file->elf, "pv_ops");
550	if (!sym)
551	return 0;
552
553	nr = sym->len / sizeof(unsigned long);
554	file->pv_ops = calloc(sizeof(struct pv_state), nr);
555	if (!file->pv_ops)
556	return -1;
557
558	for (idx = 0; idx < nr; idx++)
559	INIT_LIST_HEAD(list: &file->pv_ops[idx].targets);
560
561	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
562	add_pv_ops(file, symname: pv_ops);
563
564	return 0;
565	}
566
567	static struct instruction *find_last_insn(struct objtool_file *file,
568	struct section *sec)
569	{
570	struct instruction *insn = NULL;
571	unsigned int offset;
572	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
573
574	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
575	insn = find_insn(file, sec, offset);
576
577	return insn;
578	}
579
580	/*
581	* Mark "ud2" instructions and manually annotated dead ends.
582	*/
583	static int add_dead_ends(struct objtool_file *file)
584	{
585	struct section *rsec;
586	struct reloc *reloc;
587	struct instruction *insn;
588	uint64_t offset;
589
590	/*
591	* Check for manually annotated dead ends.
592	*/
593	rsec = find_section_by_name(file->elf, ".rela.discard.unreachable");
594	if (!rsec)
595	goto reachable;
596
597	for_each_reloc(rsec, reloc) {
598	if (reloc->sym->type == STT_SECTION) {
599	offset = reloc_addend(reloc);
600	} else if (reloc->sym->local_label) {
601	offset = reloc->sym->offset;
602	} else {
603	WARN("unexpected relocation symbol type in %s", rsec->name);
604	return -1;
605	}
606
607	insn = find_insn(file, sec: reloc->sym->sec, offset);
608	if (insn)
609	insn = prev_insn_same_sec(file, insn);
610	else if (offset == reloc->sym->sec->sh.sh_size) {
611	insn = find_last_insn(file, sec: reloc->sym->sec);
612	if (!insn) {
613	WARN("can't find unreachable insn at %s+0x%" PRIx64,
614	reloc->sym->sec->name, offset);
615	return -1;
616	}
617	} else {
618	WARN("can't find unreachable insn at %s+0x%" PRIx64,
619	reloc->sym->sec->name, offset);
620	return -1;
621	}
622
623	insn->dead_end = true;
624	}
625
626	reachable:
627	/*
628	* These manually annotated reachable checks are needed for GCC 4.4,
629	* where the Linux unreachable() macro isn't supported. In that case
630	* GCC doesn't know the "ud2" is fatal, so it generates code as if it's
631	* not a dead end.
632	*/
633	rsec = find_section_by_name(file->elf, ".rela.discard.reachable");
634	if (!rsec)
635	return 0;
636
637	for_each_reloc(rsec, reloc) {
638	if (reloc->sym->type == STT_SECTION) {
639	offset = reloc_addend(reloc);
640	} else if (reloc->sym->local_label) {
641	offset = reloc->sym->offset;
642	} else {
643	WARN("unexpected relocation symbol type in %s", rsec->name);
644	return -1;
645	}
646
647	insn = find_insn(file, sec: reloc->sym->sec, offset);
648	if (insn)
649	insn = prev_insn_same_sec(file, insn);
650	else if (offset == reloc->sym->sec->sh.sh_size) {
651	insn = find_last_insn(file, sec: reloc->sym->sec);
652	if (!insn) {
653	WARN("can't find reachable insn at %s+0x%" PRIx64,
654	reloc->sym->sec->name, offset);
655	return -1;
656	}
657	} else {
658	WARN("can't find reachable insn at %s+0x%" PRIx64,
659	reloc->sym->sec->name, offset);
660	return -1;
661	}
662
663	insn->dead_end = false;
664	}
665
666	return 0;
667	}
668
669	static int create_static_call_sections(struct objtool_file *file)
670	{
671	struct static_call_site *site;
672	struct section *sec;
673	struct instruction *insn;
674	struct symbol *key_sym;
675	char *key_name, *tmp;
676	int idx;
677
678	sec = find_section_by_name(file->elf, ".static_call_sites");
679	if (sec) {
680	INIT_LIST_HEAD(list: &file->static_call_list);
681	WARN("file already has .static_call_sites section, skipping");
682	return 0;
683	}
684
685	if (list_empty(head: &file->static_call_list))
686	return 0;
687
688	idx = 0;
689	list_for_each_entry(insn, &file->static_call_list, call_node)
690	idx++;
691
692	sec = elf_create_section_pair(file->elf, ".static_call_sites",
693	sizeof(*site), idx, idx * 2);
694	if (!sec)
695	return -1;
696
697	/* Allow modules to modify the low bits of static_call_site::key */
698	sec->sh.sh_flags |= SHF_WRITE;
699
700	idx = 0;
701	list_for_each_entry(insn, &file->static_call_list, call_node) {
702
703	/* populate reloc for 'addr' */
704	if (!elf_init_reloc_text_sym(file->elf, sec,
705	idx * sizeof(*site), idx * 2,
706	insn->sec, insn->offset))
707	return -1;
708
709	/* find key symbol */
710	key_name = strdup(insn_call_dest(insn)->name);
711	if (!key_name) {
712	perror("strdup");
713	return -1;
714	}
715	if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
716	STATIC_CALL_TRAMP_PREFIX_LEN)) {
717	WARN("static_call: trampoline name malformed: %s", key_name);
718	free(key_name);
719	return -1;
720	}
721	tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
722	memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
723
724	key_sym = find_symbol_by_name(file->elf, tmp);
725	if (!key_sym) {
726	if (!opts.module) {
727	WARN("static_call: can't find static_call_key symbol: %s", tmp);
728	free(key_name);
729	return -1;
730	}
731
732	/*
733	* For modules(), the key might not be exported, which
734	* means the module can make static calls but isn't
735	* allowed to change them.
736	*
737	* In that case we temporarily set the key to be the
738	* trampoline address. This is fixed up in
739	* static_call_add_module().
740	*/
741	key_sym = insn_call_dest(insn);
742	}
743	free(key_name);
744
745	/* populate reloc for 'key' */
746	if (!elf_init_reloc_data_sym(file->elf, sec,
747	idx * sizeof(*site) + 4,
748	(idx * 2) + 1, key_sym,
749	is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
750	return -1;
751
752	idx++;
753	}
754
755	return 0;
756	}
757
758	static int create_retpoline_sites_sections(struct objtool_file *file)
759	{
760	struct instruction *insn;
761	struct section *sec;
762	int idx;
763
764	sec = find_section_by_name(file->elf, ".retpoline_sites");
765	if (sec) {
766	WARN("file already has .retpoline_sites, skipping");
767	return 0;
768	}
769
770	idx = 0;
771	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
772	idx++;
773
774	if (!idx)
775	return 0;
776
777	sec = elf_create_section_pair(file->elf, ".retpoline_sites",
778	sizeof(int), idx, idx);
779	if (!sec)
780	return -1;
781
782	idx = 0;
783	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
784
785	if (!elf_init_reloc_text_sym(file->elf, sec,
786	idx * sizeof(int), idx,
787	insn->sec, insn->offset))
788	return -1;
789
790	idx++;
791	}
792
793	return 0;
794	}
795
796	static int create_return_sites_sections(struct objtool_file *file)
797	{
798	struct instruction *insn;
799	struct section *sec;
800	int idx;
801
802	sec = find_section_by_name(file->elf, ".return_sites");
803	if (sec) {
804	WARN("file already has .return_sites, skipping");
805	return 0;
806	}
807
808	idx = 0;
809	list_for_each_entry(insn, &file->return_thunk_list, call_node)
810	idx++;
811
812	if (!idx)
813	return 0;
814
815	sec = elf_create_section_pair(file->elf, ".return_sites",
816	sizeof(int), idx, idx);
817	if (!sec)
818	return -1;
819
820	idx = 0;
821	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
822
823	if (!elf_init_reloc_text_sym(file->elf, sec,
824	idx * sizeof(int), idx,
825	insn->sec, insn->offset))
826	return -1;
827
828	idx++;
829	}
830
831	return 0;
832	}
833
834	static int create_ibt_endbr_seal_sections(struct objtool_file *file)
835	{
836	struct instruction *insn;
837	struct section *sec;
838	int idx;
839
840	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
841	if (sec) {
842	WARN("file already has .ibt_endbr_seal, skipping");
843	return 0;
844	}
845
846	idx = 0;
847	list_for_each_entry(insn, &file->endbr_list, call_node)
848	idx++;
849
850	if (opts.stats) {
851	printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
852	printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int);
853	printf("ibt: superfluous ENDBR: %d\n", idx);
854	}
855
856	if (!idx)
857	return 0;
858
859	sec = elf_create_section_pair(file->elf, ".ibt_endbr_seal",
860	sizeof(int), idx, idx);
861	if (!sec)
862	return -1;
863
864	idx = 0;
865	list_for_each_entry(insn, &file->endbr_list, call_node) {
866
867	int *site = (int *)sec->data->d_buf + idx;
868	struct symbol *sym = insn->sym;
869	*site = 0;
870
871	if (opts.module && sym && sym->type == STT_FUNC &&
872	insn->offset == sym->offset &&
873	(!strcmp(sym->name, "init_module") ||
874	!strcmp(sym->name, "cleanup_module")))
875	WARN("%s(): not an indirect call target", sym->name);
876
877	if (!elf_init_reloc_text_sym(file->elf, sec,
878	idx * sizeof(int), idx,
879	insn->sec, insn->offset))
880	return -1;
881
882	idx++;
883	}
884
885	return 0;
886	}
887
888	static int create_cfi_sections(struct objtool_file *file)
889	{
890	struct section *sec;
891	struct symbol *sym;
892	int idx;
893
894	sec = find_section_by_name(file->elf, ".cfi_sites");
895	if (sec) {
896	INIT_LIST_HEAD(list: &file->call_list);
897	WARN("file already has .cfi_sites section, skipping");
898	return 0;
899	}
900
901	idx = 0;
902	for_each_sym(file, sym) {
903	if (sym->type != STT_FUNC)
904	continue;
905
906	if (strncmp(sym->name, "__cfi_", 6))
907	continue;
908
909	idx++;
910	}
911
912	sec = elf_create_section_pair(file->elf, ".cfi_sites",
913	sizeof(unsigned int), idx, idx);
914	if (!sec)
915	return -1;
916
917	idx = 0;
918	for_each_sym(file, sym) {
919	if (sym->type != STT_FUNC)
920	continue;
921
922	if (strncmp(sym->name, "__cfi_", 6))
923	continue;
924
925	if (!elf_init_reloc_text_sym(file->elf, sec,
926	idx * sizeof(unsigned int), idx,
927	sym->sec, sym->offset))
928	return -1;
929
930	idx++;
931	}
932
933	return 0;
934	}
935
936	static int create_mcount_loc_sections(struct objtool_file *file)
937	{
938	size_t addr_size = elf_addr_size(file->elf);
939	struct instruction *insn;
940	struct section *sec;
941	int idx;
942
943	sec = find_section_by_name(file->elf, "__mcount_loc");
944	if (sec) {
945	INIT_LIST_HEAD(list: &file->mcount_loc_list);
946	WARN("file already has __mcount_loc section, skipping");
947	return 0;
948	}
949
950	if (list_empty(head: &file->mcount_loc_list))
951	return 0;
952
953	idx = 0;
954	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
955	idx++;
956
957	sec = elf_create_section_pair(file->elf, "__mcount_loc", addr_size,
958	idx, idx);
959	if (!sec)
960	return -1;
961
962	sec->sh.sh_addralign = addr_size;
963
964	idx = 0;
965	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
966
967	struct reloc *reloc;
968
969	reloc = elf_init_reloc_text_sym(file->elf, sec, idx * addr_size, idx,
970	insn->sec, insn->offset);
971	if (!reloc)
972	return -1;
973
974	set_reloc_type(file->elf, reloc, addr_size == 8 ? R_ABS64 : R_ABS32);
975
976	idx++;
977	}
978
979	return 0;
980	}
981
982	static int create_direct_call_sections(struct objtool_file *file)
983	{
984	struct instruction *insn;
985	struct section *sec;
986	int idx;
987
988	sec = find_section_by_name(file->elf, ".call_sites");
989	if (sec) {
990	INIT_LIST_HEAD(list: &file->call_list);
991	WARN("file already has .call_sites section, skipping");
992	return 0;
993	}
994
995	if (list_empty(head: &file->call_list))
996	return 0;
997
998	idx = 0;
999	list_for_each_entry(insn, &file->call_list, call_node)
1000	idx++;
1001
1002	sec = elf_create_section_pair(file->elf, ".call_sites",
1003	sizeof(unsigned int), idx, idx);
1004	if (!sec)
1005	return -1;
1006
1007	idx = 0;
1008	list_for_each_entry(insn, &file->call_list, call_node) {
1009
1010	if (!elf_init_reloc_text_sym(file->elf, sec,
1011	idx * sizeof(unsigned int), idx,
1012	insn->sec, insn->offset))
1013	return -1;
1014
1015	idx++;
1016	}
1017
1018	return 0;
1019	}
1020
1021	/*
1022	* Warnings shouldn't be reported for ignored functions.
1023	*/
1024	static void add_ignores(struct objtool_file *file)
1025	{
1026	struct instruction *insn;
1027	struct section *rsec;
1028	struct symbol *func;
1029	struct reloc *reloc;
1030
1031	rsec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
1032	if (!rsec)
1033	return;
1034
1035	for_each_reloc(rsec, reloc) {
1036	switch (reloc->sym->type) {
1037	case STT_FUNC:
1038	func = reloc->sym;
1039	break;
1040
1041	case STT_SECTION:
1042	func = find_func_by_offset(reloc->sym->sec, reloc_addend(reloc));
1043	if (!func)
1044	continue;
1045	break;
1046
1047	default:
1048	WARN("unexpected relocation symbol type in %s: %d",
1049	rsec->name, reloc->sym->type);
1050	continue;
1051	}
1052
1053	func_for_each_insn(file, func, insn)
1054	insn->ignore = true;
1055	}
1056	}
1057
1058	/*
1059	* This is a whitelist of functions that is allowed to be called with AC set.
1060	* The list is meant to be minimal and only contains compiler instrumentation
1061	* ABI and a few functions used to implement *_{to,from}_user() functions.
1062	*
1063	* These functions must not directly change AC, but may PUSHF/POPF.
1064	*/
1065	static const char *uaccess_safe_builtin[] = {
1066	/* KASAN */
1067	"kasan_report",
1068	"kasan_check_range",
1069	/* KASAN out-of-line */
1070	"__asan_loadN_noabort",
1071	"__asan_load1_noabort",
1072	"__asan_load2_noabort",
1073	"__asan_load4_noabort",
1074	"__asan_load8_noabort",
1075	"__asan_load16_noabort",
1076	"__asan_storeN_noabort",
1077	"__asan_store1_noabort",
1078	"__asan_store2_noabort",
1079	"__asan_store4_noabort",
1080	"__asan_store8_noabort",
1081	"__asan_store16_noabort",
1082	"__kasan_check_read",
1083	"__kasan_check_write",
1084	/* KASAN in-line */
1085	"__asan_report_load_n_noabort",
1086	"__asan_report_load1_noabort",
1087	"__asan_report_load2_noabort",
1088	"__asan_report_load4_noabort",
1089	"__asan_report_load8_noabort",
1090	"__asan_report_load16_noabort",
1091	"__asan_report_store_n_noabort",
1092	"__asan_report_store1_noabort",
1093	"__asan_report_store2_noabort",
1094	"__asan_report_store4_noabort",
1095	"__asan_report_store8_noabort",
1096	"__asan_report_store16_noabort",
1097	/* KCSAN */
1098	"__kcsan_check_access",
1099	"__kcsan_mb",
1100	"__kcsan_wmb",
1101	"__kcsan_rmb",
1102	"__kcsan_release",
1103	"kcsan_found_watchpoint",
1104	"kcsan_setup_watchpoint",
1105	"kcsan_check_scoped_accesses",
1106	"kcsan_disable_current",
1107	"kcsan_enable_current_nowarn",
1108	/* KCSAN/TSAN */
1109	"__tsan_func_entry",
1110	"__tsan_func_exit",
1111	"__tsan_read_range",
1112	"__tsan_write_range",
1113	"__tsan_read1",
1114	"__tsan_read2",
1115	"__tsan_read4",
1116	"__tsan_read8",
1117	"__tsan_read16",
1118	"__tsan_write1",
1119	"__tsan_write2",
1120	"__tsan_write4",
1121	"__tsan_write8",
1122	"__tsan_write16",
1123	"__tsan_read_write1",
1124	"__tsan_read_write2",
1125	"__tsan_read_write4",
1126	"__tsan_read_write8",
1127	"__tsan_read_write16",
1128	"__tsan_volatile_read1",
1129	"__tsan_volatile_read2",
1130	"__tsan_volatile_read4",
1131	"__tsan_volatile_read8",
1132	"__tsan_volatile_read16",
1133	"__tsan_volatile_write1",
1134	"__tsan_volatile_write2",
1135	"__tsan_volatile_write4",
1136	"__tsan_volatile_write8",
1137	"__tsan_volatile_write16",
1138	"__tsan_atomic8_load",
1139	"__tsan_atomic16_load",
1140	"__tsan_atomic32_load",
1141	"__tsan_atomic64_load",
1142	"__tsan_atomic8_store",
1143	"__tsan_atomic16_store",
1144	"__tsan_atomic32_store",
1145	"__tsan_atomic64_store",
1146	"__tsan_atomic8_exchange",
1147	"__tsan_atomic16_exchange",
1148	"__tsan_atomic32_exchange",
1149	"__tsan_atomic64_exchange",
1150	"__tsan_atomic8_fetch_add",
1151	"__tsan_atomic16_fetch_add",
1152	"__tsan_atomic32_fetch_add",
1153	"__tsan_atomic64_fetch_add",
1154	"__tsan_atomic8_fetch_sub",
1155	"__tsan_atomic16_fetch_sub",
1156	"__tsan_atomic32_fetch_sub",
1157	"__tsan_atomic64_fetch_sub",
1158	"__tsan_atomic8_fetch_and",
1159	"__tsan_atomic16_fetch_and",
1160	"__tsan_atomic32_fetch_and",
1161	"__tsan_atomic64_fetch_and",
1162	"__tsan_atomic8_fetch_or",
1163	"__tsan_atomic16_fetch_or",
1164	"__tsan_atomic32_fetch_or",
1165	"__tsan_atomic64_fetch_or",
1166	"__tsan_atomic8_fetch_xor",
1167	"__tsan_atomic16_fetch_xor",
1168	"__tsan_atomic32_fetch_xor",
1169	"__tsan_atomic64_fetch_xor",
1170	"__tsan_atomic8_fetch_nand",
1171	"__tsan_atomic16_fetch_nand",
1172	"__tsan_atomic32_fetch_nand",
1173	"__tsan_atomic64_fetch_nand",
1174	"__tsan_atomic8_compare_exchange_strong",
1175	"__tsan_atomic16_compare_exchange_strong",
1176	"__tsan_atomic32_compare_exchange_strong",
1177	"__tsan_atomic64_compare_exchange_strong",
1178	"__tsan_atomic8_compare_exchange_weak",
1179	"__tsan_atomic16_compare_exchange_weak",
1180	"__tsan_atomic32_compare_exchange_weak",
1181	"__tsan_atomic64_compare_exchange_weak",
1182	"__tsan_atomic8_compare_exchange_val",
1183	"__tsan_atomic16_compare_exchange_val",
1184	"__tsan_atomic32_compare_exchange_val",
1185	"__tsan_atomic64_compare_exchange_val",
1186	"__tsan_atomic_thread_fence",
1187	"__tsan_atomic_signal_fence",
1188	"__tsan_unaligned_read16",
1189	"__tsan_unaligned_write16",
1190	/* KCOV */
1191	"write_comp_data",
1192	"check_kcov_mode",
1193	"__sanitizer_cov_trace_pc",
1194	"__sanitizer_cov_trace_const_cmp1",
1195	"__sanitizer_cov_trace_const_cmp2",
1196	"__sanitizer_cov_trace_const_cmp4",
1197	"__sanitizer_cov_trace_const_cmp8",
1198	"__sanitizer_cov_trace_cmp1",
1199	"__sanitizer_cov_trace_cmp2",
1200	"__sanitizer_cov_trace_cmp4",
1201	"__sanitizer_cov_trace_cmp8",
1202	"__sanitizer_cov_trace_switch",
1203	/* KMSAN */
1204	"kmsan_copy_to_user",
1205	"kmsan_report",
1206	"kmsan_unpoison_entry_regs",
1207	"kmsan_unpoison_memory",
1208	"__msan_chain_origin",
1209	"__msan_get_context_state",
1210	"__msan_instrument_asm_store",
1211	"__msan_metadata_ptr_for_load_1",
1212	"__msan_metadata_ptr_for_load_2",
1213	"__msan_metadata_ptr_for_load_4",
1214	"__msan_metadata_ptr_for_load_8",
1215	"__msan_metadata_ptr_for_load_n",
1216	"__msan_metadata_ptr_for_store_1",
1217	"__msan_metadata_ptr_for_store_2",
1218	"__msan_metadata_ptr_for_store_4",
1219	"__msan_metadata_ptr_for_store_8",
1220	"__msan_metadata_ptr_for_store_n",
1221	"__msan_poison_alloca",
1222	"__msan_warning",
1223	/* UBSAN */
1224	"ubsan_type_mismatch_common",
1225	"__ubsan_handle_type_mismatch",
1226	"__ubsan_handle_type_mismatch_v1",
1227	"__ubsan_handle_shift_out_of_bounds",
1228	"__ubsan_handle_load_invalid_value",
1229	/* STACKLEAK */
1230	"stackleak_track_stack",
1231	/* misc */
1232	"csum_partial_copy_generic",
1233	"copy_mc_fragile",
1234	"copy_mc_fragile_handle_tail",
1235	"copy_mc_enhanced_fast_string",
1236	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1237	"rep_stos_alternative",
1238	"rep_movs_alternative",
1239	"__copy_user_nocache",
1240	NULL
1241	};
1242
1243	static void add_uaccess_safe(struct objtool_file *file)
1244	{
1245	struct symbol *func;
1246	const char **name;
1247
1248	if (!opts.uaccess)
1249	return;
1250
1251	for (name = uaccess_safe_builtin; *name; name++) {
1252	func = find_symbol_by_name(file->elf, *name);
1253	if (!func)
1254	continue;
1255
1256	func->uaccess_safe = true;
1257	}
1258	}
1259
1260	/*
1261	* FIXME: For now, just ignore any alternatives which add retpolines. This is
1262	* a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1263	* But it at least allows objtool to understand the control flow *around* the
1264	* retpoline.
1265	*/
1266	static int add_ignore_alternatives(struct objtool_file *file)
1267	{
1268	struct section *rsec;
1269	struct reloc *reloc;
1270	struct instruction *insn;
1271
1272	rsec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1273	if (!rsec)
1274	return 0;
1275
1276	for_each_reloc(rsec, reloc) {
1277	if (reloc->sym->type != STT_SECTION) {
1278	WARN("unexpected relocation symbol type in %s", rsec->name);
1279	return -1;
1280	}
1281
1282	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
1283	if (!insn) {
1284	WARN("bad .discard.ignore_alts entry");
1285	return -1;
1286	}
1287
1288	insn->ignore_alts = true;
1289	}
1290
1291	return 0;
1292	}
1293
1294	/*
1295	* Symbols that replace INSN_CALL_DYNAMIC, every (tail) call to such a symbol
1296	* will be added to the .retpoline_sites section.
1297	*/
1298	__weak bool arch_is_retpoline(struct symbol *sym)
1299	{
1300	return false;
1301	}
1302
1303	/*
1304	* Symbols that replace INSN_RETURN, every (tail) call to such a symbol
1305	* will be added to the .return_sites section.
1306	*/
1307	__weak bool arch_is_rethunk(struct symbol *sym)
1308	{
1309	return false;
1310	}
1311
1312	/*
1313	* Symbols that are embedded inside other instructions, because sometimes crazy
1314	* code exists. These are mostly ignored for validation purposes.
1315	*/
1316	__weak bool arch_is_embedded_insn(struct symbol *sym)
1317	{
1318	return false;
1319	}
1320
1321	static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1322	{
1323	struct reloc *reloc;
1324
1325	if (insn->no_reloc)
1326	return NULL;
1327
1328	if (!file)
1329	return NULL;
1330
1331	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1332	insn->offset, insn->len);
1333	if (!reloc) {
1334	insn->no_reloc = 1;
1335	return NULL;
1336	}
1337
1338	return reloc;
1339	}
1340
1341	static void remove_insn_ops(struct instruction *insn)
1342	{
1343	struct stack_op *op, *next;
1344
1345	for (op = insn->stack_ops; op; op = next) {
1346	next = op->next;
1347	free(op);
1348	}
1349	insn->stack_ops = NULL;
1350	}
1351
1352	static void annotate_call_site(struct objtool_file *file,
1353	struct instruction *insn, bool sibling)
1354	{
1355	struct reloc *reloc = insn_reloc(file, insn);
1356	struct symbol *sym = insn_call_dest(insn);
1357
1358	if (!sym)
1359	sym = reloc->sym;
1360
1361	/*
1362	* Alternative replacement code is just template code which is
1363	* sometimes copied to the original instruction. For now, don't
1364	* annotate it. (In the future we might consider annotating the
1365	* original instruction if/when it ever makes sense to do so.)
1366	*/
1367	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1368	return;
1369
1370	if (sym->static_call_tramp) {
1371	list_add_tail(new: &insn->call_node, head: &file->static_call_list);
1372	return;
1373	}
1374
1375	if (sym->retpoline_thunk) {
1376	list_add_tail(new: &insn->call_node, head: &file->retpoline_call_list);
1377	return;
1378	}
1379
1380	/*
1381	* Many compilers cannot disable KCOV or sanitizer calls with a function
1382	* attribute so they need a little help, NOP out any such calls from
1383	* noinstr text.
1384	*/
1385	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1386	if (reloc)
1387	set_reloc_type(file->elf, reloc, R_NONE);
1388
1389	elf_write_insn(file->elf, insn->sec,
1390	insn->offset, insn->len,
1391	sibling ? arch_ret_insn(insn->len)
1392	: arch_nop_insn(insn->len));
1393
1394	insn->type = sibling ? INSN_RETURN : INSN_NOP;
1395
1396	if (sibling) {
1397	/*
1398	* We've replaced the tail-call JMP insn by two new
1399	* insn: RET; INT3, except we only have a single struct
1400	* insn here. Mark it retpoline_safe to avoid the SLS
1401	* warning, instead of adding another insn.
1402	*/
1403	insn->retpoline_safe = true;
1404	}
1405
1406	return;
1407	}
1408
1409	if (opts.mcount && sym->fentry) {
1410	if (sibling)
1411	WARN_INSN(insn, "tail call to __fentry__ !?!?");
1412	if (opts.mnop) {
1413	if (reloc)
1414	set_reloc_type(file->elf, reloc, R_NONE);
1415
1416	elf_write_insn(file->elf, insn->sec,
1417	insn->offset, insn->len,
1418	arch_nop_insn(insn->len));
1419
1420	insn->type = INSN_NOP;
1421	}
1422
1423	list_add_tail(new: &insn->call_node, head: &file->mcount_loc_list);
1424	return;
1425	}
1426
1427	if (insn->type == INSN_CALL && !insn->sec->init)
1428	list_add_tail(new: &insn->call_node, head: &file->call_list);
1429
1430	if (!sibling && dead_end_function(file, func: sym))
1431	insn->dead_end = true;
1432	}
1433
1434	static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1435	struct symbol *dest, bool sibling)
1436	{
1437	insn->_call_dest = dest;
1438	if (!dest)
1439	return;
1440
1441	/*
1442	* Whatever stack impact regular CALLs have, should be undone
1443	* by the RETURN of the called function.
1444	*
1445	* Annotated intra-function calls retain the stack_ops but
1446	* are converted to JUMP, see read_intra_function_calls().
1447	*/
1448	remove_insn_ops(insn);
1449
1450	annotate_call_site(file, insn, sibling);
1451	}
1452
1453	static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1454	{
1455	/*
1456	* Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1457	* so convert them accordingly.
1458	*/
1459	switch (insn->type) {
1460	case INSN_CALL:
1461	insn->type = INSN_CALL_DYNAMIC;
1462	break;
1463	case INSN_JUMP_UNCONDITIONAL:
1464	insn->type = INSN_JUMP_DYNAMIC;
1465	break;
1466	case INSN_JUMP_CONDITIONAL:
1467	insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1468	break;
1469	default:
1470	return;
1471	}
1472
1473	insn->retpoline_safe = true;
1474
1475	/*
1476	* Whatever stack impact regular CALLs have, should be undone
1477	* by the RETURN of the called function.
1478	*
1479	* Annotated intra-function calls retain the stack_ops but
1480	* are converted to JUMP, see read_intra_function_calls().
1481	*/
1482	remove_insn_ops(insn);
1483
1484	annotate_call_site(file, insn, sibling: false);
1485	}
1486
1487	static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1488	{
1489	/*
1490	* Return thunk tail calls are really just returns in disguise,
1491	* so convert them accordingly.
1492	*/
1493	insn->type = INSN_RETURN;
1494	insn->retpoline_safe = true;
1495
1496	if (add)
1497	list_add_tail(new: &insn->call_node, head: &file->return_thunk_list);
1498	}
1499
1500	static bool is_first_func_insn(struct objtool_file *file,
1501	struct instruction *insn, struct symbol *sym)
1502	{
1503	if (insn->offset == sym->offset)
1504	return true;
1505
1506	/* Allow direct CALL/JMP past ENDBR */
1507	if (opts.ibt) {
1508	struct instruction *prev = prev_insn_same_sym(file, insn);
1509
1510	if (prev && prev->type == INSN_ENDBR &&
1511	insn->offset == sym->offset + prev->len)
1512	return true;
1513	}
1514
1515	return false;
1516	}
1517
1518	/*
1519	* A sibling call is a tail-call to another symbol -- to differentiate from a
1520	* recursive tail-call which is to the same symbol.
1521	*/
1522	static bool jump_is_sibling_call(struct objtool_file *file,
1523	struct instruction *from, struct instruction *to)
1524	{
1525	struct symbol *fs = from->sym;
1526	struct symbol *ts = to->sym;
1527
1528	/* Not a sibling call if from/to a symbol hole */
1529	if (!fs || !ts)
1530	return false;
1531
1532	/* Not a sibling call if not targeting the start of a symbol. */
1533	if (!is_first_func_insn(file, insn: to, sym: ts))
1534	return false;
1535
1536	/* Disallow sibling calls into STT_NOTYPE */
1537	if (ts->type == STT_NOTYPE)
1538	return false;
1539
1540	/* Must not be self to be a sibling */
1541	return fs->pfunc != ts->pfunc;
1542	}
1543
1544	/*
1545	* Find the destination instructions for all jumps.
1546	*/
1547	static int add_jump_destinations(struct objtool_file *file)
1548	{
1549	struct instruction *insn, *jump_dest;
1550	struct reloc *reloc;
1551	struct section *dest_sec;
1552	unsigned long dest_off;
1553
1554	for_each_insn(file, insn) {
1555	if (insn->jump_dest) {
1556	/*
1557	* handle_group_alt() may have previously set
1558	* 'jump_dest' for some alternatives.
1559	*/
1560	continue;
1561	}
1562	if (!is_static_jump(insn))
1563	continue;
1564
1565	reloc = insn_reloc(file, insn);
1566	if (!reloc) {
1567	dest_sec = insn->sec;
1568	dest_off = arch_jump_destination(insn);
1569	} else if (reloc->sym->type == STT_SECTION) {
1570	dest_sec = reloc->sym->sec;
1571	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1572	} else if (reloc->sym->retpoline_thunk) {
1573	add_retpoline_call(file, insn);
1574	continue;
1575	} else if (reloc->sym->return_thunk) {
1576	add_return_call(file, insn, true);
1577	continue;
1578	} else if (insn_func(insn)) {
1579	/*
1580	* External sibling call or internal sibling call with
1581	* STT_FUNC reloc.
1582	*/
1583	add_call_dest(file, insn, reloc->sym, true);
1584	continue;
1585	} else if (reloc->sym->sec->idx) {
1586	dest_sec = reloc->sym->sec;
1587	dest_off = reloc->sym->sym.st_value +
1588	arch_dest_reloc_offset(reloc_addend(reloc));
1589	} else {
1590	/* non-func asm code jumping to another file */
1591	continue;
1592	}
1593
1594	jump_dest = find_insn(file, dest_sec, dest_off);
1595	if (!jump_dest) {
1596	struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1597
1598	/*
1599	* This is a special case for retbleed_untrain_ret().
1600	* It jumps to __x86_return_thunk(), but objtool
1601	* can't find the thunk's starting RET
1602	* instruction, because the RET is also in the
1603	* middle of another instruction. Objtool only
1604	* knows about the outer instruction.
1605	*/
1606	if (sym && sym->embedded_insn) {
1607	add_return_call(file, insn, false);
1608	continue;
1609	}
1610
1611	WARN_INSN(insn, "can't find jump dest instruction at %s+0x%lx",
1612	dest_sec->name, dest_off);
1613	return -1;
1614	}
1615
1616	/*
1617	* An intra-TU jump in retpoline.o might not have a relocation
1618	* for its jump dest, in which case the above
1619	* add_{retpoline,return}_call() didn't happen.
1620	*/
1621	if (jump_dest->sym && jump_dest->offset == jump_dest->sym->offset) {
1622	if (jump_dest->sym->retpoline_thunk) {
1623	add_retpoline_call(file, insn);
1624	continue;
1625	}
1626	if (jump_dest->sym->return_thunk) {
1627	add_return_call(file, insn, true);
1628	continue;
1629	}
1630	}
1631
1632	/*
1633	* Cross-function jump.
1634	*/
1635	if (insn_func(insn) && insn_func(jump_dest) &&
1636	insn_func(insn) != insn_func(jump_dest)) {
1637
1638	/*
1639	* For GCC 8+, create parent/child links for any cold
1640	* subfunctions. This is _mostly_ redundant with a
1641	* similar initialization in read_symbols().
1642	*
1643	* If a function has aliases, we want the *first* such
1644	* function in the symbol table to be the subfunction's
1645	* parent. In that case we overwrite the
1646	* initialization done in read_symbols().
1647	*
1648	* However this code can't completely replace the
1649	* read_symbols() code because this doesn't detect the
1650	* case where the parent function's only reference to a
1651	* subfunction is through a jump table.
1652	*/
1653	if (!strstr(insn_func(insn)->name, ".cold") &&
1654	strstr(insn_func(jump_dest)->name, ".cold")) {
1655	insn_func(insn)->cfunc = insn_func(jump_dest);
1656	insn_func(jump_dest)->pfunc = insn_func(insn);
1657	}
1658	}
1659
1660	if (jump_is_sibling_call(file, insn, jump_dest)) {
1661	/*
1662	* Internal sibling call without reloc or with
1663	* STT_SECTION reloc.
1664	*/
1665	add_call_dest(file, insn, insn_func(jump_dest), true);
1666	continue;
1667	}
1668
1669	insn->jump_dest = jump_dest;
1670	}
1671
1672	return 0;
1673	}
1674
1675	static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1676	{
1677	struct symbol *call_dest;
1678
1679	call_dest = find_func_by_offset(sec, offset);
1680	if (!call_dest)
1681	call_dest = find_symbol_by_offset(sec, offset);
1682
1683	return call_dest;
1684	}
1685
1686	/*
1687	* Find the destination instructions for all calls.
1688	*/
1689	static int add_call_destinations(struct objtool_file *file)
1690	{
1691	struct instruction *insn;
1692	unsigned long dest_off;
1693	struct symbol *dest;
1694	struct reloc *reloc;
1695
1696	for_each_insn(file, insn) {
1697	if (insn->type != INSN_CALL)
1698	continue;
1699
1700	reloc = insn_reloc(file, insn);
1701	if (!reloc) {
1702	dest_off = arch_jump_destination(insn);
1703	dest = find_call_destination(insn->sec, dest_off);
1704
1705	add_call_dest(file, insn, dest, false);
1706
1707	if (insn->ignore)
1708	continue;
1709
1710	if (!insn_call_dest(insn)) {
1711	WARN_INSN(insn, "unannotated intra-function call");
1712	return -1;
1713	}
1714
1715	if (insn_func(insn) && insn_call_dest(insn)->type != STT_FUNC) {
1716	WARN_INSN(insn, "unsupported call to non-function");
1717	return -1;
1718	}
1719
1720	} else if (reloc->sym->type == STT_SECTION) {
1721	dest_off = arch_dest_reloc_offset(reloc_addend(reloc));
1722	dest = find_call_destination(reloc->sym->sec, dest_off);
1723	if (!dest) {
1724	WARN_INSN(insn, "can't find call dest symbol at %s+0x%lx",
1725	reloc->sym->sec->name, dest_off);
1726	return -1;
1727	}
1728
1729	add_call_dest(file, insn, dest, false);
1730
1731	} else if (reloc->sym->retpoline_thunk) {
1732	add_retpoline_call(file, insn);
1733
1734	} else
1735	add_call_dest(file, insn, reloc->sym, false);
1736	}
1737
1738	return 0;
1739	}
1740
1741	/*
1742	* The .alternatives section requires some extra special care over and above
1743	* other special sections because alternatives are patched in place.
1744	*/
1745	static int handle_group_alt(struct objtool_file *file,
1746	struct special_alt *special_alt,
1747	struct instruction *orig_insn,
1748	struct instruction **new_insn)
1749	{
1750	struct instruction *last_new_insn = NULL, *insn, *nop = NULL;
1751	struct alt_group *orig_alt_group, *new_alt_group;
1752	unsigned long dest_off;
1753
1754	orig_alt_group = orig_insn->alt_group;
1755	if (!orig_alt_group) {
1756	struct instruction *last_orig_insn = NULL;
1757
1758	orig_alt_group = malloc(sizeof(*orig_alt_group));
1759	if (!orig_alt_group) {
1760	WARN("malloc failed");
1761	return -1;
1762	}
1763	orig_alt_group->cfi = calloc(special_alt->orig_len,
1764	sizeof(struct cfi_state *));
1765	if (!orig_alt_group->cfi) {
1766	WARN("calloc failed");
1767	return -1;
1768	}
1769
1770	insn = orig_insn;
1771	sec_for_each_insn_from(file, insn) {
1772	if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1773	break;
1774
1775	insn->alt_group = orig_alt_group;
1776	last_orig_insn = insn;
1777	}
1778	orig_alt_group->orig_group = NULL;
1779	orig_alt_group->first_insn = orig_insn;
1780	orig_alt_group->last_insn = last_orig_insn;
1781	orig_alt_group->nop = NULL;
1782	} else {
1783	if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
1784	orig_alt_group->first_insn->offset != special_alt->orig_len) {
1785	WARN_INSN(orig_insn, "weirdly overlapping alternative! %ld != %d",
1786	orig_alt_group->last_insn->offset +
1787	orig_alt_group->last_insn->len -
1788	orig_alt_group->first_insn->offset,
1789	special_alt->orig_len);
1790	return -1;
1791	}
1792	}
1793
1794	new_alt_group = malloc(sizeof(*new_alt_group));
1795	if (!new_alt_group) {
1796	WARN("malloc failed");
1797	return -1;
1798	}
1799
1800	if (special_alt->new_len < special_alt->orig_len) {
1801	/*
1802	* Insert a fake nop at the end to make the replacement
1803	* alt_group the same size as the original. This is needed to
1804	* allow propagate_alt_cfi() to do its magic. When the last
1805	* instruction affects the stack, the instruction after it (the
1806	* nop) will propagate the new state to the shared CFI array.
1807	*/
1808	nop = malloc(sizeof(*nop));
1809	if (!nop) {
1810	WARN("malloc failed");
1811	return -1;
1812	}
1813	memset(nop, 0, sizeof(*nop));
1814
1815	nop->sec = special_alt->new_sec;
1816	nop->offset = special_alt->new_off + special_alt->new_len;
1817	nop->len = special_alt->orig_len - special_alt->new_len;
1818	nop->type = INSN_NOP;
1819	nop->sym = orig_insn->sym;
1820	nop->alt_group = new_alt_group;
1821	nop->ignore = orig_insn->ignore_alts;
1822	}
1823
1824	if (!special_alt->new_len) {
1825	*new_insn = nop;
1826	goto end;
1827	}
1828
1829	insn = *new_insn;
1830	sec_for_each_insn_from(file, insn) {
1831	struct reloc *alt_reloc;
1832
1833	if (insn->offset >= special_alt->new_off + special_alt->new_len)
1834	break;
1835
1836	last_new_insn = insn;
1837
1838	insn->ignore = orig_insn->ignore_alts;
1839	insn->sym = orig_insn->sym;
1840	insn->alt_group = new_alt_group;
1841
1842	/*
1843	* Since alternative replacement code is copy/pasted by the
1844	* kernel after applying relocations, generally such code can't
1845	* have relative-address relocation references to outside the
1846	* .altinstr_replacement section, unless the arch's
1847	* alternatives code can adjust the relative offsets
1848	* accordingly.
1849	*/
1850	alt_reloc = insn_reloc(file, insn);
1851	if (alt_reloc && arch_pc_relative_reloc(alt_reloc) &&
1852	!arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1853
1854	WARN_INSN(insn, "unsupported relocation in alternatives section");
1855	return -1;
1856	}
1857
1858	if (!is_static_jump(insn))
1859	continue;
1860
1861	if (!insn->immediate)
1862	continue;
1863
1864	dest_off = arch_jump_destination(insn);
1865	if (dest_off == special_alt->new_off + special_alt->new_len) {
1866	insn->jump_dest = next_insn_same_sec(file, insn: orig_alt_group->last_insn);
1867	if (!insn->jump_dest) {
1868	WARN_INSN(insn, "can't find alternative jump destination");
1869	return -1;
1870	}
1871	}
1872	}
1873
1874	if (!last_new_insn) {
1875	WARN_FUNC("can't find last new alternative instruction",
1876	special_alt->new_sec, special_alt->new_off);
1877	return -1;
1878	}
1879
1880	end:
1881	new_alt_group->orig_group = orig_alt_group;
1882	new_alt_group->first_insn = *new_insn;
1883	new_alt_group->last_insn = last_new_insn;
1884	new_alt_group->nop = nop;
1885	new_alt_group->cfi = orig_alt_group->cfi;
1886	return 0;
1887	}
1888
1889	/*
1890	* A jump table entry can either convert a nop to a jump or a jump to a nop.
1891	* If the original instruction is a jump, make the alt entry an effective nop
1892	* by just skipping the original instruction.
1893	*/
1894	static int handle_jump_alt(struct objtool_file *file,
1895	struct special_alt *special_alt,
1896	struct instruction *orig_insn,
1897	struct instruction **new_insn)
1898	{
1899	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1900	orig_insn->type != INSN_NOP) {
1901
1902	WARN_INSN(orig_insn, "unsupported instruction at jump label");
1903	return -1;
1904	}
1905
1906	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1907	struct reloc *reloc = insn_reloc(file, insn: orig_insn);
1908
1909	if (reloc)
1910	set_reloc_type(file->elf, reloc, R_NONE);
1911	elf_write_insn(file->elf, orig_insn->sec,
1912	orig_insn->offset, orig_insn->len,
1913	arch_nop_insn(orig_insn->len));
1914	orig_insn->type = INSN_NOP;
1915	}
1916
1917	if (orig_insn->type == INSN_NOP) {
1918	if (orig_insn->len == 2)
1919	file->jl_nop_short++;
1920	else
1921	file->jl_nop_long++;
1922
1923	return 0;
1924	}
1925
1926	if (orig_insn->len == 2)
1927	file->jl_short++;
1928	else
1929	file->jl_long++;
1930
1931	*new_insn = next_insn_same_sec(file, insn: orig_insn);
1932	return 0;
1933	}
1934
1935	/*
1936	* Read all the special sections which have alternate instructions which can be
1937	* patched in or redirected to at runtime. Each instruction having alternate
1938	* instruction(s) has them added to its insn->alts list, which will be
1939	* traversed in validate_branch().
1940	*/
1941	static int add_special_section_alts(struct objtool_file *file)
1942	{
1943	struct list_head special_alts;
1944	struct instruction *orig_insn, *new_insn;
1945	struct special_alt *special_alt, *tmp;
1946	struct alternative *alt;
1947	int ret;
1948
1949	ret = special_get_alts(file->elf, &special_alts);
1950	if (ret)
1951	return ret;
1952
1953	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1954
1955	orig_insn = find_insn(file, sec: special_alt->orig_sec,
1956	offset: special_alt->orig_off);
1957	if (!orig_insn) {
1958	WARN_FUNC("special: can't find orig instruction",
1959	special_alt->orig_sec, special_alt->orig_off);
1960	ret = -1;
1961	goto out;
1962	}
1963
1964	new_insn = NULL;
1965	if (!special_alt->group || special_alt->new_len) {
1966	new_insn = find_insn(file, sec: special_alt->new_sec,
1967	offset: special_alt->new_off);
1968	if (!new_insn) {
1969	WARN_FUNC("special: can't find new instruction",
1970	special_alt->new_sec,
1971	special_alt->new_off);
1972	ret = -1;
1973	goto out;
1974	}
1975	}
1976
1977	if (special_alt->group) {
1978	if (!special_alt->orig_len) {
1979	WARN_INSN(orig_insn, "empty alternative entry");
1980	continue;
1981	}
1982
1983	ret = handle_group_alt(file, special_alt, orig_insn,
1984	new_insn: &new_insn);
1985	if (ret)
1986	goto out;
1987	} else if (special_alt->jump_or_nop) {
1988	ret = handle_jump_alt(file, special_alt, orig_insn,
1989	new_insn: &new_insn);
1990	if (ret)
1991	goto out;
1992	}
1993
1994	alt = malloc(sizeof(*alt));
1995	if (!alt) {
1996	WARN("malloc failed");
1997	ret = -1;
1998	goto out;
1999	}
2000
2001	alt->insn = new_insn;
2002	alt->skip_orig = special_alt->skip_orig;
2003	orig_insn->ignore_alts |= special_alt->skip_alt;
2004	alt->next = orig_insn->alts;
2005	orig_insn->alts = alt;
2006
2007	list_del(entry: &special_alt->list);
2008	free(special_alt);
2009	}
2010
2011	if (opts.stats) {
2012	printf("jl\\\tNOP\tJMP\n");
2013	printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
2014	printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
2015	}
2016
2017	out:
2018	return ret;
2019	}
2020
2021	static int add_jump_table(struct objtool_file *file, struct instruction *insn,
2022	struct reloc *next_table)
2023	{
2024	struct symbol *pfunc = insn_func(insn)->pfunc;
2025	struct reloc *table = insn_jump_table(insn);
2026	struct instruction *dest_insn;
2027	unsigned int prev_offset = 0;
2028	struct reloc *reloc = table;
2029	struct alternative *alt;
2030
2031	/*
2032	* Each @reloc is a switch table relocation which points to the target
2033	* instruction.
2034	*/
2035	for_each_reloc_from(table->sec, reloc) {
2036
2037	/* Check for the end of the table: */
2038	if (reloc != table && reloc == next_table)
2039	break;
2040
2041	/* Make sure the table entries are consecutive: */
2042	if (prev_offset && reloc_offset(reloc) != prev_offset + 8)
2043	break;
2044
2045	/* Detect function pointers from contiguous objects: */
2046	if (reloc->sym->sec == pfunc->sec &&
2047	reloc_addend(reloc) == pfunc->offset)
2048	break;
2049
2050	dest_insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2051	if (!dest_insn)
2052	break;
2053
2054	/* Make sure the destination is in the same function: */
2055	if (!insn_func(dest_insn) || insn_func(dest_insn)->pfunc != pfunc)
2056	break;
2057
2058	alt = malloc(sizeof(*alt));
2059	if (!alt) {
2060	WARN("malloc failed");
2061	return -1;
2062	}
2063
2064	alt->insn = dest_insn;
2065	alt->next = insn->alts;
2066	insn->alts = alt;
2067	prev_offset = reloc_offset(reloc);
2068	}
2069
2070	if (!prev_offset) {
2071	WARN_INSN(insn, "can't find switch jump table");
2072	return -1;
2073	}
2074
2075	return 0;
2076	}
2077
2078	/*
2079	* find_jump_table() - Given a dynamic jump, find the switch jump table
2080	* associated with it.
2081	*/
2082	static struct reloc *find_jump_table(struct objtool_file *file,
2083	struct symbol *func,
2084	struct instruction *insn)
2085	{
2086	struct reloc *table_reloc;
2087	struct instruction *dest_insn, *orig_insn = insn;
2088
2089	/*
2090	* Backward search using the @first_jump_src links, these help avoid
2091	* much of the 'in between' code. Which avoids us getting confused by
2092	* it.
2093	*/
2094	for (;
2095	insn && insn_func(insn) && insn_func(insn)->pfunc == func;
2096	insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
2097
2098	if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
2099	break;
2100
2101	/* allow small jumps within the range */
2102	if (insn->type == INSN_JUMP_UNCONDITIONAL &&
2103	insn->jump_dest &&
2104	(insn->jump_dest->offset <= insn->offset ||
2105	insn->jump_dest->offset > orig_insn->offset))
2106	break;
2107
2108	table_reloc = arch_find_switch_table(file, insn);
2109	if (!table_reloc)
2110	continue;
2111	dest_insn = find_insn(file, sec: table_reloc->sym->sec, offset: reloc_addend(table_reloc));
2112	if (!dest_insn || !insn_func(dest_insn) || insn_func(dest_insn)->pfunc != func)
2113	continue;
2114
2115	return table_reloc;
2116	}
2117
2118	return NULL;
2119	}
2120
2121	/*
2122	* First pass: Mark the head of each jump table so that in the next pass,
2123	* we know when a given jump table ends and the next one starts.
2124	*/
2125	static void mark_func_jump_tables(struct objtool_file *file,
2126	struct symbol *func)
2127	{
2128	struct instruction *insn, *last = NULL;
2129	struct reloc *reloc;
2130
2131	func_for_each_insn(file, func, insn) {
2132	if (!last)
2133	last = insn;
2134
2135	/*
2136	* Store back-pointers for unconditional forward jumps such
2137	* that find_jump_table() can back-track using those and
2138	* avoid some potentially confusing code.
2139	*/
2140	if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
2141	insn->offset > last->offset &&
2142	insn->jump_dest->offset > insn->offset &&
2143	!insn->jump_dest->first_jump_src) {
2144
2145	insn->jump_dest->first_jump_src = insn;
2146	last = insn->jump_dest;
2147	}
2148
2149	if (insn->type != INSN_JUMP_DYNAMIC)
2150	continue;
2151
2152	reloc = find_jump_table(file, func, insn);
2153	if (reloc)
2154	insn->_jump_table = reloc;
2155	}
2156	}
2157
2158	static int add_func_jump_tables(struct objtool_file *file,
2159	struct symbol *func)
2160	{
2161	struct instruction *insn, *insn_t1 = NULL, *insn_t2;
2162	int ret = 0;
2163
2164	func_for_each_insn(file, func, insn) {
2165	if (!insn_jump_table(insn))
2166	continue;
2167
2168	if (!insn_t1) {
2169	insn_t1 = insn;
2170	continue;
2171	}
2172
2173	insn_t2 = insn;
2174
2175	ret = add_jump_table(file, insn: insn_t1, next_table: insn_jump_table(insn: insn_t2));
2176	if (ret)
2177	return ret;
2178
2179	insn_t1 = insn_t2;
2180	}
2181
2182	if (insn_t1)
2183	ret = add_jump_table(file, insn: insn_t1, NULL);
2184
2185	return ret;
2186	}
2187
2188	/*
2189	* For some switch statements, gcc generates a jump table in the .rodata
2190	* section which contains a list of addresses within the function to jump to.
2191	* This finds these jump tables and adds them to the insn->alts lists.
2192	*/
2193	static int add_jump_table_alts(struct objtool_file *file)
2194	{
2195	struct symbol *func;
2196	int ret;
2197
2198	if (!file->rodata)
2199	return 0;
2200
2201	for_each_sym(file, func) {
2202	if (func->type != STT_FUNC)
2203	continue;
2204
2205	mark_func_jump_tables(file, func);
2206	ret = add_func_jump_tables(file, func);
2207	if (ret)
2208	return ret;
2209	}
2210
2211	return 0;
2212	}
2213
2214	static void set_func_state(struct cfi_state *state)
2215	{
2216	state->cfa = initial_func_cfi.cfa;
2217	memcpy(&state->regs, &initial_func_cfi.regs,
2218	CFI_NUM_REGS * sizeof(struct cfi_reg));
2219	state->stack_size = initial_func_cfi.cfa.offset;
2220	state->type = UNWIND_HINT_TYPE_CALL;
2221	}
2222
2223	static int read_unwind_hints(struct objtool_file *file)
2224	{
2225	struct cfi_state cfi = init_cfi;
2226	struct section *sec;
2227	struct unwind_hint *hint;
2228	struct instruction *insn;
2229	struct reloc *reloc;
2230	unsigned long offset;
2231	int i;
2232
2233	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2234	if (!sec)
2235	return 0;
2236
2237	if (!sec->rsec) {
2238	WARN("missing .rela.discard.unwind_hints section");
2239	return -1;
2240	}
2241
2242	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2243	WARN("struct unwind_hint size mismatch");
2244	return -1;
2245	}
2246
2247	file->hints = true;
2248
2249	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2250	hint = (struct unwind_hint *)sec->data->d_buf + i;
2251
2252	reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2253	if (!reloc) {
2254	WARN("can't find reloc for unwind_hints[%d]", i);
2255	return -1;
2256	}
2257
2258	if (reloc->sym->type == STT_SECTION) {
2259	offset = reloc_addend(reloc);
2260	} else if (reloc->sym->local_label) {
2261	offset = reloc->sym->offset;
2262	} else {
2263	WARN("unexpected relocation symbol type in %s", sec->rsec->name);
2264	return -1;
2265	}
2266
2267	insn = find_insn(file, sec: reloc->sym->sec, offset);
2268	if (!insn) {
2269	WARN("can't find insn for unwind_hints[%d]", i);
2270	return -1;
2271	}
2272
2273	insn->hint = true;
2274
2275	if (hint->type == UNWIND_HINT_TYPE_UNDEFINED) {
2276	insn->cfi = &force_undefined_cfi;
2277	continue;
2278	}
2279
2280	if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2281	insn->hint = false;
2282	insn->save = true;
2283	continue;
2284	}
2285
2286	if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2287	insn->restore = true;
2288	continue;
2289	}
2290
2291	if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2292	struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2293
2294	if (sym && sym->bind == STB_GLOBAL) {
2295	if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2296	WARN_INSN(insn, "UNWIND_HINT_IRET_REGS without ENDBR");
2297	}
2298	}
2299	}
2300
2301	if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2302	insn->cfi = &func_cfi;
2303	continue;
2304	}
2305
2306	if (insn->cfi)
2307	cfi = *(insn->cfi);
2308
2309	if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2310	WARN_INSN(insn, "unsupported unwind_hint sp base reg %d", hint->sp_reg);
2311	return -1;
2312	}
2313
2314	cfi.cfa.offset = bswap_if_needed(file->elf, hint->sp_offset);
2315	cfi.type = hint->type;
2316	cfi.signal = hint->signal;
2317
2318	insn->cfi = cfi_hash_find_or_add(cfi: &cfi);
2319	}
2320
2321	return 0;
2322	}
2323
2324	static int read_noendbr_hints(struct objtool_file *file)
2325	{
2326	struct instruction *insn;
2327	struct section *rsec;
2328	struct reloc *reloc;
2329
2330	rsec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2331	if (!rsec)
2332	return 0;
2333
2334	for_each_reloc(rsec, reloc) {
2335	insn = find_insn(file, sec: reloc->sym->sec,
2336	offset: reloc->sym->offset + reloc_addend(reloc));
2337	if (!insn) {
2338	WARN("bad .discard.noendbr entry");
2339	return -1;
2340	}
2341
2342	insn->noendbr = 1;
2343	}
2344
2345	return 0;
2346	}
2347
2348	static int read_retpoline_hints(struct objtool_file *file)
2349	{
2350	struct section *rsec;
2351	struct instruction *insn;
2352	struct reloc *reloc;
2353
2354	rsec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2355	if (!rsec)
2356	return 0;
2357
2358	for_each_reloc(rsec, reloc) {
2359	if (reloc->sym->type != STT_SECTION) {
2360	WARN("unexpected relocation symbol type in %s", rsec->name);
2361	return -1;
2362	}
2363
2364	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2365	if (!insn) {
2366	WARN("bad .discard.retpoline_safe entry");
2367	return -1;
2368	}
2369
2370	if (insn->type != INSN_JUMP_DYNAMIC &&
2371	insn->type != INSN_CALL_DYNAMIC &&
2372	insn->type != INSN_RETURN &&
2373	insn->type != INSN_NOP) {
2374	WARN_INSN(insn, "retpoline_safe hint not an indirect jump/call/ret/nop");
2375	return -1;
2376	}
2377
2378	insn->retpoline_safe = true;
2379	}
2380
2381	return 0;
2382	}
2383
2384	static int read_instr_hints(struct objtool_file *file)
2385	{
2386	struct section *rsec;
2387	struct instruction *insn;
2388	struct reloc *reloc;
2389
2390	rsec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2391	if (!rsec)
2392	return 0;
2393
2394	for_each_reloc(rsec, reloc) {
2395	if (reloc->sym->type != STT_SECTION) {
2396	WARN("unexpected relocation symbol type in %s", rsec->name);
2397	return -1;
2398	}
2399
2400	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2401	if (!insn) {
2402	WARN("bad .discard.instr_end entry");
2403	return -1;
2404	}
2405
2406	insn->instr--;
2407	}
2408
2409	rsec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2410	if (!rsec)
2411	return 0;
2412
2413	for_each_reloc(rsec, reloc) {
2414	if (reloc->sym->type != STT_SECTION) {
2415	WARN("unexpected relocation symbol type in %s", rsec->name);
2416	return -1;
2417	}
2418
2419	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2420	if (!insn) {
2421	WARN("bad .discard.instr_begin entry");
2422	return -1;
2423	}
2424
2425	insn->instr++;
2426	}
2427
2428	return 0;
2429	}
2430
2431	static int read_validate_unret_hints(struct objtool_file *file)
2432	{
2433	struct section *rsec;
2434	struct instruction *insn;
2435	struct reloc *reloc;
2436
2437	rsec = find_section_by_name(file->elf, ".rela.discard.validate_unret");
2438	if (!rsec)
2439	return 0;
2440
2441	for_each_reloc(rsec, reloc) {
2442	if (reloc->sym->type != STT_SECTION) {
2443	WARN("unexpected relocation symbol type in %s", rsec->name);
2444	return -1;
2445	}
2446
2447	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2448	if (!insn) {
2449	WARN("bad .discard.instr_end entry");
2450	return -1;
2451	}
2452	insn->unret = 1;
2453	}
2454
2455	return 0;
2456	}
2457
2458
2459	static int read_intra_function_calls(struct objtool_file *file)
2460	{
2461	struct instruction *insn;
2462	struct section *rsec;
2463	struct reloc *reloc;
2464
2465	rsec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2466	if (!rsec)
2467	return 0;
2468
2469	for_each_reloc(rsec, reloc) {
2470	unsigned long dest_off;
2471
2472	if (reloc->sym->type != STT_SECTION) {
2473	WARN("unexpected relocation symbol type in %s",
2474	rsec->name);
2475	return -1;
2476	}
2477
2478	insn = find_insn(file, sec: reloc->sym->sec, offset: reloc_addend(reloc));
2479	if (!insn) {
2480	WARN("bad .discard.intra_function_call entry");
2481	return -1;
2482	}
2483
2484	if (insn->type != INSN_CALL) {
2485	WARN_INSN(insn, "intra_function_call not a direct call");
2486	return -1;
2487	}
2488
2489	/*
2490	* Treat intra-function CALLs as JMPs, but with a stack_op.
2491	* See add_call_destinations(), which strips stack_ops from
2492	* normal CALLs.
2493	*/
2494	insn->type = INSN_JUMP_UNCONDITIONAL;
2495
2496	dest_off = arch_jump_destination(insn);
2497	insn->jump_dest = find_insn(file, sec: insn->sec, offset: dest_off);
2498	if (!insn->jump_dest) {
2499	WARN_INSN(insn, "can't find call dest at %s+0x%lx",
2500	insn->sec->name, dest_off);
2501	return -1;
2502	}
2503	}
2504
2505	return 0;
2506	}
2507
2508	/*
2509	* Return true if name matches an instrumentation function, where calls to that
2510	* function from noinstr code can safely be removed, but compilers won't do so.
2511	*/
2512	static bool is_profiling_func(const char *name)
2513	{
2514	/*
2515	* Many compilers cannot disable KCOV with a function attribute.
2516	*/
2517	if (!strncmp(name, "__sanitizer_cov_", 16))
2518	return true;
2519
2520	/*
2521	* Some compilers currently do not remove __tsan_func_entry/exit nor
2522	* __tsan_atomic_signal_fence (used for barrier instrumentation) with
2523	* the __no_sanitize_thread attribute, remove them. Once the kernel's
2524	* minimum Clang version is 14.0, this can be removed.
2525	*/
2526	if (!strncmp(name, "__tsan_func_", 12) ||
2527	!strcmp(name, "__tsan_atomic_signal_fence"))
2528	return true;
2529
2530	return false;
2531	}
2532
2533	static int classify_symbols(struct objtool_file *file)
2534	{
2535	struct symbol *func;
2536
2537	for_each_sym(file, func) {
2538	if (func->type == STT_NOTYPE && strstarts(func->name, ".L"))
2539	func->local_label = true;
2540
2541	if (func->bind != STB_GLOBAL)
2542	continue;
2543
2544	if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2545	strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2546	func->static_call_tramp = true;
2547
2548	if (arch_is_retpoline(sym: func))
2549	func->retpoline_thunk = true;
2550
2551	if (arch_is_rethunk(sym: func))
2552	func->return_thunk = true;
2553
2554	if (arch_is_embedded_insn(sym: func))
2555	func->embedded_insn = true;
2556
2557	if (arch_ftrace_match(func->name))
2558	func->fentry = true;
2559
2560	if (is_profiling_func(name: func->name))
2561	func->profiling_func = true;
2562	}
2563
2564	return 0;
2565	}
2566
2567	static void mark_rodata(struct objtool_file *file)
2568	{
2569	struct section *sec;
2570	bool found = false;
2571
2572	/*
2573	* Search for the following rodata sections, each of which can
2574	* potentially contain jump tables:
2575	*
2576	* - .rodata: can contain GCC switch tables
2577	* - .rodata.<func>: same, if -fdata-sections is being used
2578	* - .rodata..c_jump_table: contains C annotated jump tables
2579	*
2580	* .rodata.str1.* sections are ignored; they don't contain jump tables.
2581	*/
2582	for_each_sec(file, sec) {
2583	if (!strncmp(sec->name, ".rodata", 7) &&
2584	!strstr(sec->name, ".str1.")) {
2585	sec->rodata = true;
2586	found = true;
2587	}
2588	}
2589
2590	file->rodata = found;
2591	}
2592
2593	static int decode_sections(struct objtool_file *file)
2594	{
2595	int ret;
2596
2597	mark_rodata(file);
2598
2599	ret = init_pv_ops(file);
2600	if (ret)
2601	return ret;
2602
2603	/*
2604	* Must be before add_{jump_call}_destination.
2605	*/
2606	ret = classify_symbols(file);
2607	if (ret)
2608	return ret;
2609
2610	ret = decode_instructions(file);
2611	if (ret)
2612	return ret;
2613
2614	add_ignores(file);
2615	add_uaccess_safe(file);
2616
2617	ret = add_ignore_alternatives(file);
2618	if (ret)
2619	return ret;
2620
2621	/*
2622	* Must be before read_unwind_hints() since that needs insn->noendbr.
2623	*/
2624	ret = read_noendbr_hints(file);
2625	if (ret)
2626	return ret;
2627
2628	/*
2629	* Must be before add_jump_destinations(), which depends on 'func'
2630	* being set for alternatives, to enable proper sibling call detection.
2631	*/
2632	if (opts.stackval || opts.orc || opts.uaccess || opts.noinstr) {
2633	ret = add_special_section_alts(file);
2634	if (ret)
2635	return ret;
2636	}
2637
2638	ret = add_jump_destinations(file);
2639	if (ret)
2640	return ret;
2641
2642	/*
2643	* Must be before add_call_destination(); it changes INSN_CALL to
2644	* INSN_JUMP.
2645	*/
2646	ret = read_intra_function_calls(file);
2647	if (ret)
2648	return ret;
2649
2650	ret = add_call_destinations(file);
2651	if (ret)
2652	return ret;
2653
2654	/*
2655	* Must be after add_call_destinations() such that it can override
2656	* dead_end_function() marks.
2657	*/
2658	ret = add_dead_ends(file);
2659	if (ret)
2660	return ret;
2661
2662	ret = add_jump_table_alts(file);
2663	if (ret)
2664	return ret;
2665
2666	ret = read_unwind_hints(file);
2667	if (ret)
2668	return ret;
2669
2670	ret = read_retpoline_hints(file);
2671	if (ret)
2672	return ret;
2673
2674	ret = read_instr_hints(file);
2675	if (ret)
2676	return ret;
2677
2678	ret = read_validate_unret_hints(file);
2679	if (ret)
2680	return ret;
2681
2682	return 0;
2683	}
2684
2685	static bool is_special_call(struct instruction *insn)
2686	{
2687	if (insn->type == INSN_CALL) {
2688	struct symbol *dest = insn_call_dest(insn);
2689
2690	if (!dest)
2691	return false;
2692
2693	if (dest->fentry || dest->embedded_insn)
2694	return true;
2695	}
2696
2697	return false;
2698	}
2699
2700	static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2701	{
2702	struct cfi_state *cfi = &state->cfi;
2703	int i;
2704
2705	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2706	return true;
2707
2708	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2709	return true;
2710
2711	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2712	return true;
2713
2714	for (i = 0; i < CFI_NUM_REGS; i++) {
2715	if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2716	cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2717	return true;
2718	}
2719
2720	return false;
2721	}
2722
2723	static bool check_reg_frame_pos(const struct cfi_reg *reg,
2724	int expected_offset)
2725	{
2726	return reg->base == CFI_CFA &&
2727	reg->offset == expected_offset;
2728	}
2729
2730	static bool has_valid_stack_frame(struct insn_state *state)
2731	{
2732	struct cfi_state *cfi = &state->cfi;
2733
2734	if (cfi->cfa.base == CFI_BP &&
2735	check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2736	check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2737	return true;
2738
2739	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2740	return true;
2741
2742	return false;
2743	}
2744
2745	static int update_cfi_state_regs(struct instruction *insn,
2746	struct cfi_state *cfi,
2747	struct stack_op *op)
2748	{
2749	struct cfi_reg *cfa = &cfi->cfa;
2750
2751	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2752	return 0;
2753
2754	/* push */
2755	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2756	cfa->offset += 8;
2757
2758	/* pop */
2759	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2760	cfa->offset -= 8;
2761
2762	/* add immediate to sp */
2763	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2764	op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2765	cfa->offset -= op->src.offset;
2766
2767	return 0;
2768	}
2769
2770	static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2771	{
2772	if (arch_callee_saved_reg(reg) &&
2773	cfi->regs[reg].base == CFI_UNDEFINED) {
2774	cfi->regs[reg].base = base;
2775	cfi->regs[reg].offset = offset;
2776	}
2777	}
2778
2779	static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2780	{
2781	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2782	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2783	}
2784
2785	/*
2786	* A note about DRAP stack alignment:
2787	*
2788	* GCC has the concept of a DRAP register, which is used to help keep track of
2789	* the stack pointer when aligning the stack. r10 or r13 is used as the DRAP
2790	* register. The typical DRAP pattern is:
2791	*
2792	* 4c 8d 54 24 08 lea 0x8(%rsp),%r10
2793	* 48 83 e4 c0 and $0xffffffffffffffc0,%rsp
2794	* 41 ff 72 f8 pushq -0x8(%r10)
2795	* 55 push %rbp
2796	* 48 89 e5 mov %rsp,%rbp
2797	* (more pushes)
2798	* 41 52 push %r10
2799	* ...
2800	* 41 5a pop %r10
2801	* (more pops)
2802	* 5d pop %rbp
2803	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2804	* c3 retq
2805	*
2806	* There are some variations in the epilogues, like:
2807	*
2808	* 5b pop %rbx
2809	* 41 5a pop %r10
2810	* 41 5c pop %r12
2811	* 41 5d pop %r13
2812	* 41 5e pop %r14
2813	* c9 leaveq
2814	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2815	* c3 retq
2816	*
2817	* and:
2818	*
2819	* 4c 8b 55 e8 mov -0x18(%rbp),%r10
2820	* 48 8b 5d e0 mov -0x20(%rbp),%rbx
2821	* 4c 8b 65 f0 mov -0x10(%rbp),%r12
2822	* 4c 8b 6d f8 mov -0x8(%rbp),%r13
2823	* c9 leaveq
2824	* 49 8d 62 f8 lea -0x8(%r10),%rsp
2825	* c3 retq
2826	*
2827	* Sometimes r13 is used as the DRAP register, in which case it's saved and
2828	* restored beforehand:
2829	*
2830	* 41 55 push %r13
2831	* 4c 8d 6c 24 10 lea 0x10(%rsp),%r13
2832	* 48 83 e4 f0 and $0xfffffffffffffff0,%rsp
2833	* ...
2834	* 49 8d 65 f0 lea -0x10(%r13),%rsp
2835	* 41 5d pop %r13
2836	* c3 retq
2837	*/
2838	static int update_cfi_state(struct instruction *insn,
2839	struct instruction *next_insn,
2840	struct cfi_state *cfi, struct stack_op *op)
2841	{
2842	struct cfi_reg *cfa = &cfi->cfa;
2843	struct cfi_reg *regs = cfi->regs;
2844
2845	/* ignore UNWIND_HINT_UNDEFINED regions */
2846	if (cfi->force_undefined)
2847	return 0;
2848
2849	/* stack operations don't make sense with an undefined CFA */
2850	if (cfa->base == CFI_UNDEFINED) {
2851	if (insn_func(insn)) {
2852	WARN_INSN(insn, "undefined stack state");
2853	return -1;
2854	}
2855	return 0;
2856	}
2857
2858	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2859	cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2860	return update_cfi_state_regs(insn, cfi, op);
2861
2862	switch (op->dest.type) {
2863
2864	case OP_DEST_REG:
2865	switch (op->src.type) {
2866
2867	case OP_SRC_REG:
2868	if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2869	cfa->base == CFI_SP &&
2870	check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2871
2872	/* mov %rsp, %rbp */
2873	cfa->base = op->dest.reg;
2874	cfi->bp_scratch = false;
2875	}
2876
2877	else if (op->src.reg == CFI_SP &&
2878	op->dest.reg == CFI_BP && cfi->drap) {
2879
2880	/* drap: mov %rsp, %rbp */
2881	regs[CFI_BP].base = CFI_BP;
2882	regs[CFI_BP].offset = -cfi->stack_size;
2883	cfi->bp_scratch = false;
2884	}
2885
2886	else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2887
2888	/*
2889	* mov %rsp, %reg
2890	*
2891	* This is needed for the rare case where GCC
2892	* does:
2893	*
2894	* mov %rsp, %rax
2895	* ...
2896	* mov %rax, %rsp
2897	*/
2898	cfi->vals[op->dest.reg].base = CFI_CFA;
2899	cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2900	}
2901
2902	else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2903	(cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2904
2905	/*
2906	* mov %rbp, %rsp
2907	*
2908	* Restore the original stack pointer (Clang).
2909	*/
2910	cfi->stack_size = -cfi->regs[CFI_BP].offset;
2911	}
2912
2913	else if (op->dest.reg == cfa->base) {
2914
2915	/* mov %reg, %rsp */
2916	if (cfa->base == CFI_SP &&
2917	cfi->vals[op->src.reg].base == CFI_CFA) {
2918
2919	/*
2920	* This is needed for the rare case
2921	* where GCC does something dumb like:
2922	*
2923	* lea 0x8(%rsp), %rcx
2924	* ...
2925	* mov %rcx, %rsp
2926	*/
2927	cfa->offset = -cfi->vals[op->src.reg].offset;
2928	cfi->stack_size = cfa->offset;
2929
2930	} else if (cfa->base == CFI_SP &&
2931	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2932	cfi->vals[op->src.reg].offset == cfa->offset) {
2933
2934	/*
2935	* Stack swizzle:
2936	*
2937	* 1: mov %rsp, (%[tos])
2938	* 2: mov %[tos], %rsp
2939	* ...
2940	* 3: pop %rsp
2941	*
2942	* Where:
2943	*
2944	* 1 - places a pointer to the previous
2945	* stack at the Top-of-Stack of the
2946	* new stack.
2947	*
2948	* 2 - switches to the new stack.
2949	*
2950	* 3 - pops the Top-of-Stack to restore
2951	* the original stack.
2952	*
2953	* Note: we set base to SP_INDIRECT
2954	* here and preserve offset. Therefore
2955	* when the unwinder reaches ToS it
2956	* will dereference SP and then add the
2957	* offset to find the next frame, IOW:
2958	* (%rsp) + offset.
2959	*/
2960	cfa->base = CFI_SP_INDIRECT;
2961
2962	} else {
2963	cfa->base = CFI_UNDEFINED;
2964	cfa->offset = 0;
2965	}
2966	}
2967
2968	else if (op->dest.reg == CFI_SP &&
2969	cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2970	cfi->vals[op->src.reg].offset == cfa->offset) {
2971
2972	/*
2973	* The same stack swizzle case 2) as above. But
2974	* because we can't change cfa->base, case 3)
2975	* will become a regular POP. Pretend we're a
2976	* PUSH so things don't go unbalanced.
2977	*/
2978	cfi->stack_size += 8;
2979	}
2980
2981
2982	break;
2983
2984	case OP_SRC_ADD:
2985	if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2986
2987	/* add imm, %rsp */
2988	cfi->stack_size -= op->src.offset;
2989	if (cfa->base == CFI_SP)
2990	cfa->offset -= op->src.offset;
2991	break;
2992	}
2993
2994	if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2995
2996	/* lea disp(%rbp), %rsp */
2997	cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2998	break;
2999	}
3000
3001	if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
3002
3003	/* drap: lea disp(%rsp), %drap */
3004	cfi->drap_reg = op->dest.reg;
3005
3006	/*
3007	* lea disp(%rsp), %reg
3008	*
3009	* This is needed for the rare case where GCC
3010	* does something dumb like:
3011	*
3012	* lea 0x8(%rsp), %rcx
3013	* ...
3014	* mov %rcx, %rsp
3015	*/
3016	cfi->vals[op->dest.reg].base = CFI_CFA;
3017	cfi->vals[op->dest.reg].offset = \
3018	-cfi->stack_size + op->src.offset;
3019
3020	break;
3021	}
3022
3023	if (cfi->drap && op->dest.reg == CFI_SP &&
3024	op->src.reg == cfi->drap_reg) {
3025
3026	/* drap: lea disp(%drap), %rsp */
3027	cfa->base = CFI_SP;
3028	cfa->offset = cfi->stack_size = -op->src.offset;
3029	cfi->drap_reg = CFI_UNDEFINED;
3030	cfi->drap = false;
3031	break;
3032	}
3033
3034	if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
3035	WARN_INSN(insn, "unsupported stack register modification");
3036	return -1;
3037	}
3038
3039	break;
3040
3041	case OP_SRC_AND:
3042	if (op->dest.reg != CFI_SP ||
3043	(cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
3044	(cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
3045	WARN_INSN(insn, "unsupported stack pointer realignment");
3046	return -1;
3047	}
3048
3049	if (cfi->drap_reg != CFI_UNDEFINED) {
3050	/* drap: and imm, %rsp */
3051	cfa->base = cfi->drap_reg;
3052	cfa->offset = cfi->stack_size = 0;
3053	cfi->drap = true;
3054	}
3055
3056	/*
3057	* Older versions of GCC (4.8ish) realign the stack
3058	* without DRAP, with a frame pointer.
3059	*/
3060
3061	break;
3062
3063	case OP_SRC_POP:
3064	case OP_SRC_POPF:
3065	if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
3066
3067	/* pop %rsp; # restore from a stack swizzle */
3068	cfa->base = CFI_SP;
3069	break;
3070	}
3071
3072	if (!cfi->drap && op->dest.reg == cfa->base) {
3073
3074	/* pop %rbp */
3075	cfa->base = CFI_SP;
3076	}
3077
3078	if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
3079	op->dest.reg == cfi->drap_reg &&
3080	cfi->drap_offset == -cfi->stack_size) {
3081
3082	/* drap: pop %drap */
3083	cfa->base = cfi->drap_reg;
3084	cfa->offset = 0;
3085	cfi->drap_offset = -1;
3086
3087	} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
3088
3089	/* pop %reg */
3090	restore_reg(cfi, reg: op->dest.reg);
3091	}
3092
3093	cfi->stack_size -= 8;
3094	if (cfa->base == CFI_SP)
3095	cfa->offset -= 8;
3096
3097	break;
3098
3099	case OP_SRC_REG_INDIRECT:
3100	if (!cfi->drap && op->dest.reg == cfa->base &&
3101	op->dest.reg == CFI_BP) {
3102
3103	/* mov disp(%rsp), %rbp */
3104	cfa->base = CFI_SP;
3105	cfa->offset = cfi->stack_size;
3106	}
3107
3108	if (cfi->drap && op->src.reg == CFI_BP &&
3109	op->src.offset == cfi->drap_offset) {
3110
3111	/* drap: mov disp(%rbp), %drap */
3112	cfa->base = cfi->drap_reg;
3113	cfa->offset = 0;
3114	cfi->drap_offset = -1;
3115	}
3116
3117	if (cfi->drap && op->src.reg == CFI_BP &&
3118	op->src.offset == regs[op->dest.reg].offset) {
3119
3120	/* drap: mov disp(%rbp), %reg */
3121	restore_reg(cfi, reg: op->dest.reg);
3122
3123	} else if (op->src.reg == cfa->base &&
3124	op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
3125
3126	/* mov disp(%rbp), %reg */
3127	/* mov disp(%rsp), %reg */
3128	restore_reg(cfi, reg: op->dest.reg);
3129
3130	} else if (op->src.reg == CFI_SP &&
3131	op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
3132
3133	/* mov disp(%rsp), %reg */
3134	restore_reg(cfi, reg: op->dest.reg);
3135	}
3136
3137	break;
3138
3139	default:
3140	WARN_INSN(insn, "unknown stack-related instruction");
3141	return -1;
3142	}
3143
3144	break;
3145
3146	case OP_DEST_PUSH:
3147	case OP_DEST_PUSHF:
3148	cfi->stack_size += 8;
3149	if (cfa->base == CFI_SP)
3150	cfa->offset += 8;
3151
3152	if (op->src.type != OP_SRC_REG)
3153	break;
3154
3155	if (cfi->drap) {
3156	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3157
3158	/* drap: push %drap */
3159	cfa->base = CFI_BP_INDIRECT;
3160	cfa->offset = -cfi->stack_size;
3161
3162	/* save drap so we know when to restore it */
3163	cfi->drap_offset = -cfi->stack_size;
3164
3165	} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
3166
3167	/* drap: push %rbp */
3168	cfi->stack_size = 0;
3169
3170	} else {
3171
3172	/* drap: push %reg */
3173	save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
3174	}
3175
3176	} else {
3177
3178	/* push %reg */
3179	save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
3180	}
3181
3182	/* detect when asm code uses rbp as a scratch register */
3183	if (opts.stackval && insn_func(insn) && op->src.reg == CFI_BP &&
3184	cfa->base != CFI_BP)
3185	cfi->bp_scratch = true;
3186	break;
3187
3188	case OP_DEST_REG_INDIRECT:
3189
3190	if (cfi->drap) {
3191	if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
3192
3193	/* drap: mov %drap, disp(%rbp) */
3194	cfa->base = CFI_BP_INDIRECT;
3195	cfa->offset = op->dest.offset;
3196
3197	/* save drap offset so we know when to restore it */
3198	cfi->drap_offset = op->dest.offset;
3199	} else {
3200
3201	/* drap: mov reg, disp(%rbp) */
3202	save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
3203	}
3204
3205	} else if (op->dest.reg == cfa->base) {
3206
3207	/* mov reg, disp(%rbp) */
3208	/* mov reg, disp(%rsp) */
3209	save_reg(cfi, op->src.reg, CFI_CFA,
3210	op->dest.offset - cfi->cfa.offset);
3211
3212	} else if (op->dest.reg == CFI_SP) {
3213
3214	/* mov reg, disp(%rsp) */
3215	save_reg(cfi, op->src.reg, CFI_CFA,
3216	op->dest.offset - cfi->stack_size);
3217
3218	} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
3219
3220	/* mov %rsp, (%reg); # setup a stack swizzle. */
3221	cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
3222	cfi->vals[op->dest.reg].offset = cfa->offset;
3223	}
3224
3225	break;
3226
3227	case OP_DEST_MEM:
3228	if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
3229	WARN_INSN(insn, "unknown stack-related memory operation");
3230	return -1;
3231	}
3232
3233	/* pop mem */
3234	cfi->stack_size -= 8;
3235	if (cfa->base == CFI_SP)
3236	cfa->offset -= 8;
3237
3238	break;
3239
3240	default:
3241	WARN_INSN(insn, "unknown stack-related instruction");
3242	return -1;
3243	}
3244
3245	return 0;
3246	}
3247
3248	/*
3249	* The stack layouts of alternatives instructions can sometimes diverge when
3250	* they have stack modifications. That's fine as long as the potential stack
3251	* layouts don't conflict at any given potential instruction boundary.
3252	*
3253	* Flatten the CFIs of the different alternative code streams (both original
3254	* and replacement) into a single shared CFI array which can be used to detect
3255	* conflicts and nicely feed a linear array of ORC entries to the unwinder.
3256	*/
3257	static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3258	{
3259	struct cfi_state **alt_cfi;
3260	int group_off;
3261
3262	if (!insn->alt_group)
3263	return 0;
3264
3265	if (!insn->cfi) {
3266	WARN("CFI missing");
3267	return -1;
3268	}
3269
3270	alt_cfi = insn->alt_group->cfi;
3271	group_off = insn->offset - insn->alt_group->first_insn->offset;
3272
3273	if (!alt_cfi[group_off]) {
3274	alt_cfi[group_off] = insn->cfi;
3275	} else {
3276	if (cficmp(cfi1: alt_cfi[group_off], cfi2: insn->cfi)) {
3277	struct alt_group *orig_group = insn->alt_group->orig_group ?: insn->alt_group;
3278	struct instruction *orig = orig_group->first_insn;
3279	char *where = offstr(insn->sec, insn->offset);
3280	WARN_INSN(orig, "stack layout conflict in alternatives: %s", where);
3281	free(where);
3282	return -1;
3283	}
3284	}
3285
3286	return 0;
3287	}
3288
3289	static int handle_insn_ops(struct instruction *insn,
3290	struct instruction *next_insn,
3291	struct insn_state *state)
3292	{
3293	struct stack_op *op;
3294
3295	for (op = insn->stack_ops; op; op = op->next) {
3296
3297	if (update_cfi_state(insn, next_insn, cfi: &state->cfi, op))
3298	return 1;
3299
3300	if (!insn->alt_group)
3301	continue;
3302
3303	if (op->dest.type == OP_DEST_PUSHF) {
3304	if (!state->uaccess_stack) {
3305	state->uaccess_stack = 1;
3306	} else if (state->uaccess_stack >> 31) {
3307	WARN_INSN(insn, "PUSHF stack exhausted");
3308	return 1;
3309	}
3310	state->uaccess_stack <<= 1;
3311	state->uaccess_stack |= state->uaccess;
3312	}
3313
3314	if (op->src.type == OP_SRC_POPF) {
3315	if (state->uaccess_stack) {
3316	state->uaccess = state->uaccess_stack & 1;
3317	state->uaccess_stack >>= 1;
3318	if (state->uaccess_stack == 1)
3319	state->uaccess_stack = 0;
3320	}
3321	}
3322	}
3323
3324	return 0;
3325	}
3326
3327	static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3328	{
3329	struct cfi_state *cfi1 = insn->cfi;
3330	int i;
3331
3332	if (!cfi1) {
3333	WARN("CFI missing");
3334	return false;
3335	}
3336
3337	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3338
3339	WARN_INSN(insn, "stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3340	cfi1->cfa.base, cfi1->cfa.offset,
3341	cfi2->cfa.base, cfi2->cfa.offset);
3342
3343	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3344	for (i = 0; i < CFI_NUM_REGS; i++) {
3345	if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3346	sizeof(struct cfi_reg)))
3347	continue;
3348
3349	WARN_INSN(insn, "stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3350	i, cfi1->regs[i].base, cfi1->regs[i].offset,
3351	i, cfi2->regs[i].base, cfi2->regs[i].offset);
3352	break;
3353	}
3354
3355	} else if (cfi1->type != cfi2->type) {
3356
3357	WARN_INSN(insn, "stack state mismatch: type1=%d type2=%d",
3358	cfi1->type, cfi2->type);
3359
3360	} else if (cfi1->drap != cfi2->drap ||
3361	(cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3362	(cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3363
3364	WARN_INSN(insn, "stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3365	cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3366	cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3367
3368	} else
3369	return true;
3370
3371	return false;
3372	}
3373
3374	static inline bool func_uaccess_safe(struct symbol *func)
3375	{
3376	if (func)
3377	return func->uaccess_safe;
3378
3379	return false;
3380	}
3381
3382	static inline const char *call_dest_name(struct instruction *insn)
3383	{
3384	static char pvname[19];
3385	struct reloc *reloc;
3386	int idx;
3387
3388	if (insn_call_dest(insn))
3389	return insn_call_dest(insn)->name;
3390
3391	reloc = insn_reloc(NULL, insn);
3392	if (reloc && !strcmp(reloc->sym->name, "pv_ops")) {
3393	idx = (reloc_addend(reloc) / sizeof(void *));
3394	snprintf(buf: pvname, size: sizeof(pvname), fmt: "pv_ops[%d]", idx);
3395	return pvname;
3396	}
3397
3398	return "{dynamic}";
3399	}
3400
3401	static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3402	{
3403	struct symbol *target;
3404	struct reloc *reloc;
3405	int idx;
3406
3407	reloc = insn_reloc(file, insn);
3408	if (!reloc || strcmp(reloc->sym->name, "pv_ops"))
3409	return false;
3410
3411	idx = (arch_dest_reloc_offset(reloc_addend(reloc)) / sizeof(void *));
3412
3413	if (file->pv_ops[idx].clean)
3414	return true;
3415
3416	file->pv_ops[idx].clean = true;
3417
3418	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3419	if (!target->sec->noinstr) {
3420	WARN("pv_ops[%d]: %s", idx, target->name);
3421	file->pv_ops[idx].clean = false;
3422	}
3423	}
3424
3425	return file->pv_ops[idx].clean;
3426	}
3427
3428	static inline bool noinstr_call_dest(struct objtool_file *file,
3429	struct instruction *insn,
3430	struct symbol *func)
3431	{
3432	/*
3433	* We can't deal with indirect function calls at present;
3434	* assume they're instrumented.
3435	*/
3436	if (!func) {
3437	if (file->pv_ops)
3438	return pv_call_dest(file, insn);
3439
3440	return false;
3441	}
3442
3443	/*
3444	* If the symbol is from a noinstr section; we good.
3445	*/
3446	if (func->sec->noinstr)
3447	return true;
3448
3449	/*
3450	* If the symbol is a static_call trampoline, we can't tell.
3451	*/
3452	if (func->static_call_tramp)
3453	return true;
3454
3455	/*
3456	* The __ubsan_handle_*() calls are like WARN(), they only happen when
3457	* something 'BAD' happened. At the risk of taking the machine down,
3458	* let them proceed to get the message out.
3459	*/
3460	if (!strncmp(func->name, "__ubsan_handle_", 15))
3461	return true;
3462
3463	return false;
3464	}
3465
3466	static int validate_call(struct objtool_file *file,
3467	struct instruction *insn,
3468	struct insn_state *state)
3469	{
3470	if (state->noinstr && state->instr <= 0 &&
3471	!noinstr_call_dest(file, insn, insn_call_dest(insn))) {
3472	WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn));
3473	return 1;
3474	}
3475
3476	if (state->uaccess && !func_uaccess_safe(func: insn_call_dest(insn))) {
3477	WARN_INSN(insn, "call to %s() with UACCESS enabled", call_dest_name(insn));
3478	return 1;
3479	}
3480
3481	if (state->df) {
3482	WARN_INSN(insn, "call to %s() with DF set", call_dest_name(insn));
3483	return 1;
3484	}
3485
3486	return 0;
3487	}
3488
3489	static int validate_sibling_call(struct objtool_file *file,
3490	struct instruction *insn,
3491	struct insn_state *state)
3492	{
3493	if (insn_func(insn) && has_modified_stack_frame(insn, state)) {
3494	WARN_INSN(insn, "sibling call from callable instruction with modified stack frame");
3495	return 1;
3496	}
3497
3498	return validate_call(file, insn, state);
3499	}
3500
3501	static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3502	{
3503	if (state->noinstr && state->instr > 0) {
3504	WARN_INSN(insn, "return with instrumentation enabled");
3505	return 1;
3506	}
3507
3508	if (state->uaccess && !func_uaccess_safe(func)) {
3509	WARN_INSN(insn, "return with UACCESS enabled");
3510	return 1;
3511	}
3512
3513	if (!state->uaccess && func_uaccess_safe(func)) {
3514	WARN_INSN(insn, "return with UACCESS disabled from a UACCESS-safe function");
3515	return 1;
3516	}
3517
3518	if (state->df) {
3519	WARN_INSN(insn, "return with DF set");
3520	return 1;
3521	}
3522
3523	if (func && has_modified_stack_frame(insn, state)) {
3524	WARN_INSN(insn, "return with modified stack frame");
3525	return 1;
3526	}
3527
3528	if (state->cfi.bp_scratch) {
3529	WARN_INSN(insn, "BP used as a scratch register");
3530	return 1;
3531	}
3532
3533	return 0;
3534	}
3535
3536	static struct instruction *next_insn_to_validate(struct objtool_file *file,
3537	struct instruction *insn)
3538	{
3539	struct alt_group *alt_group = insn->alt_group;
3540
3541	/*
3542	* Simulate the fact that alternatives are patched in-place. When the
3543	* end of a replacement alt_group is reached, redirect objtool flow to
3544	* the end of the original alt_group.
3545	*
3546	* insn->alts->insn -> alt_group->first_insn
3547	* ...
3548	* alt_group->last_insn
3549	* [alt_group->nop] -> next(orig_group->last_insn)
3550	*/
3551	if (alt_group) {
3552	if (alt_group->nop) {
3553	/* ->nop implies ->orig_group */
3554	if (insn == alt_group->last_insn)
3555	return alt_group->nop;
3556	if (insn == alt_group->nop)
3557	goto next_orig;
3558	}
3559	if (insn == alt_group->last_insn && alt_group->orig_group)
3560	goto next_orig;
3561	}
3562
3563	return next_insn_same_sec(file, insn);
3564
3565	next_orig:
3566	return next_insn_same_sec(file, insn: alt_group->orig_group->last_insn);
3567	}
3568
3569	/*
3570	* Follow the branch starting at the given instruction, and recursively follow
3571	* any other branches (jumps). Meanwhile, track the frame pointer state at
3572	* each instruction and validate all the rules described in
3573	* tools/objtool/Documentation/objtool.txt.
3574	*/
3575	static int validate_branch(struct objtool_file *file, struct symbol *func,
3576	struct instruction *insn, struct insn_state state)
3577	{
3578	struct alternative *alt;
3579	struct instruction *next_insn, *prev_insn = NULL;
3580	struct section *sec;
3581	u8 visited;
3582	int ret;
3583
3584	sec = insn->sec;
3585
3586	while (1) {
3587	next_insn = next_insn_to_validate(file, insn);
3588
3589	if (func && insn_func(insn) && func != insn_func(insn)->pfunc) {
3590	/* Ignore KCFI type preambles, which always fall through */
3591	if (!strncmp(func->name, "__cfi_", 6) ||
3592	!strncmp(func->name, "__pfx_", 6))
3593	return 0;
3594
3595	WARN("%s() falls through to next function %s()",
3596	func->name, insn_func(insn)->name);
3597	return 1;
3598	}
3599
3600	if (func && insn->ignore) {
3601	WARN_INSN(insn, "BUG: why am I validating an ignored function?");
3602	return 1;
3603	}
3604
3605	visited = VISITED_BRANCH << state.uaccess;
3606	if (insn->visited & VISITED_BRANCH_MASK) {
3607	if (!insn->hint && !insn_cfi_match(insn, cfi2: &state.cfi))
3608	return 1;
3609
3610	if (insn->visited & visited)
3611	return 0;
3612	} else {
3613	nr_insns_visited++;
3614	}
3615
3616	if (state.noinstr)
3617	state.instr += insn->instr;
3618
3619	if (insn->hint) {
3620	if (insn->restore) {
3621	struct instruction *save_insn, *i;
3622
3623	i = insn;
3624	save_insn = NULL;
3625
3626	sym_for_each_insn_continue_reverse(file, func, i) {
3627	if (i->save) {
3628	save_insn = i;
3629	break;
3630	}
3631	}
3632
3633	if (!save_insn) {
3634	WARN_INSN(insn, "no corresponding CFI save for CFI restore");
3635	return 1;
3636	}
3637
3638	if (!save_insn->visited) {
3639	/*
3640	* If the restore hint insn is at the
3641	* beginning of a basic block and was
3642	* branched to from elsewhere, and the
3643	* save insn hasn't been visited yet,
3644	* defer following this branch for now.
3645	* It will be seen later via the
3646	* straight-line path.
3647	*/
3648	if (!prev_insn)
3649	return 0;
3650
3651	WARN_INSN(insn, "objtool isn't smart enough to handle this CFI save/restore combo");
3652	return 1;
3653	}
3654
3655	insn->cfi = save_insn->cfi;
3656	nr_cfi_reused++;
3657	}
3658
3659	state.cfi = *insn->cfi;
3660	} else {
3661	/* XXX track if we actually changed state.cfi */
3662
3663	if (prev_insn && !cficmp(cfi1: prev_insn->cfi, cfi2: &state.cfi)) {
3664	insn->cfi = prev_insn->cfi;
3665	nr_cfi_reused++;
3666	} else {
3667	insn->cfi = cfi_hash_find_or_add(cfi: &state.cfi);
3668	}
3669	}
3670
3671	insn->visited |= visited;
3672
3673	if (propagate_alt_cfi(file, insn))
3674	return 1;
3675
3676	if (!insn->ignore_alts && insn->alts) {
3677	bool skip_orig = false;
3678
3679	for (alt = insn->alts; alt; alt = alt->next) {
3680	if (alt->skip_orig)
3681	skip_orig = true;
3682
3683	ret = validate_branch(file, func, insn: alt->insn, state: state);
3684	if (ret) {
3685	BT_INSN(insn, "(alt)");
3686	return ret;
3687	}
3688	}
3689
3690	if (skip_orig)
3691	return 0;
3692	}
3693
3694	if (handle_insn_ops(insn, next_insn, state: &state))
3695	return 1;
3696
3697	switch (insn->type) {
3698
3699	case INSN_RETURN:
3700	return validate_return(func, insn, state: &state);
3701
3702	case INSN_CALL:
3703	case INSN_CALL_DYNAMIC:
3704	ret = validate_call(file, insn, state: &state);
3705	if (ret)
3706	return ret;
3707
3708	if (opts.stackval && func && !is_special_call(insn) &&
3709	!has_valid_stack_frame(&state)) {
3710	WARN_INSN(insn, "call without frame pointer save/setup");
3711	return 1;
3712	}
3713
3714	if (insn->dead_end)
3715	return 0;
3716
3717	break;
3718
3719	case INSN_JUMP_CONDITIONAL:
3720	case INSN_JUMP_UNCONDITIONAL:
3721	if (is_sibling_call(insn)) {
3722	ret = validate_sibling_call(file, insn, state: &state);
3723	if (ret)
3724	return ret;
3725
3726	} else if (insn->jump_dest) {
3727	ret = validate_branch(file, func,
3728	insn: insn->jump_dest, state: state);
3729	if (ret) {
3730	BT_INSN(insn, "(branch)");
3731	return ret;
3732	}
3733	}
3734
3735	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3736	return 0;
3737
3738	break;
3739
3740	case INSN_JUMP_DYNAMIC:
3741	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3742	if (is_sibling_call(insn)) {
3743	ret = validate_sibling_call(file, insn, state: &state);
3744	if (ret)
3745	return ret;
3746	}
3747
3748	if (insn->type == INSN_JUMP_DYNAMIC)
3749	return 0;
3750
3751	break;
3752
3753	case INSN_CONTEXT_SWITCH:
3754	if (func && (!next_insn || !next_insn->hint)) {
3755	WARN_INSN(insn, "unsupported instruction in callable function");
3756	return 1;
3757	}
3758	return 0;
3759
3760	case INSN_STAC:
3761	if (state.uaccess) {
3762	WARN_INSN(insn, "recursive UACCESS enable");
3763	return 1;
3764	}
3765
3766	state.uaccess = true;
3767	break;
3768
3769	case INSN_CLAC:
3770	if (!state.uaccess && func) {
3771	WARN_INSN(insn, "redundant UACCESS disable");
3772	return 1;
3773	}
3774
3775	if (func_uaccess_safe(func) && !state.uaccess_stack) {
3776	WARN_INSN(insn, "UACCESS-safe disables UACCESS");
3777	return 1;
3778	}
3779
3780	state.uaccess = false;
3781	break;
3782
3783	case INSN_STD:
3784	if (state.df) {
3785	WARN_INSN(insn, "recursive STD");
3786	return 1;
3787	}
3788
3789	state.df = true;
3790	break;
3791
3792	case INSN_CLD:
3793	if (!state.df && func) {
3794	WARN_INSN(insn, "redundant CLD");
3795	return 1;
3796	}
3797
3798	state.df = false;
3799	break;
3800
3801	default:
3802	break;
3803	}
3804
3805	if (insn->dead_end)
3806	return 0;
3807
3808	if (!next_insn) {
3809	if (state.cfi.cfa.base == CFI_UNDEFINED)
3810	return 0;
3811	WARN("%s: unexpected end of section", sec->name);
3812	return 1;
3813	}
3814
3815	prev_insn = insn;
3816	insn = next_insn;
3817	}
3818
3819	return 0;
3820	}
3821
3822	static int validate_unwind_hint(struct objtool_file *file,
3823	struct instruction *insn,
3824	struct insn_state *state)
3825	{
3826	if (insn->hint && !insn->visited && !insn->ignore) {
3827	int ret = validate_branch(file, insn_func(insn), insn, *state);
3828	if (ret)
3829	BT_INSN(insn, "<=== (hint)");
3830	return ret;
3831	}
3832
3833	return 0;
3834	}
3835
3836	static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3837	{
3838	struct instruction *insn;
3839	struct insn_state state;
3840	int warnings = 0;
3841
3842	if (!file->hints)
3843	return 0;
3844
3845	init_insn_state(file, state: &state, sec);
3846
3847	if (sec) {
3848	sec_for_each_insn(file, sec, insn)
3849	warnings += validate_unwind_hint(file, insn, state: &state);
3850	} else {
3851	for_each_insn(file, insn)
3852	warnings += validate_unwind_hint(file, insn, &state);
3853	}
3854
3855	return warnings;
3856	}
3857
3858	/*
3859	* Validate rethunk entry constraint: must untrain RET before the first RET.
3860	*
3861	* Follow every branch (intra-function) and ensure VALIDATE_UNRET_END comes
3862	* before an actual RET instruction.
3863	*/
3864	static int validate_unret(struct objtool_file *file, struct instruction *insn)
3865	{
3866	struct instruction *next, *dest;
3867	int ret;
3868
3869	for (;;) {
3870	next = next_insn_to_validate(file, insn);
3871
3872	if (insn->visited & VISITED_UNRET)
3873	return 0;
3874
3875	insn->visited |= VISITED_UNRET;
3876
3877	if (!insn->ignore_alts && insn->alts) {
3878	struct alternative *alt;
3879	bool skip_orig = false;
3880
3881	for (alt = insn->alts; alt; alt = alt->next) {
3882	if (alt->skip_orig)
3883	skip_orig = true;
3884
3885	ret = validate_unret(file, insn: alt->insn);
3886	if (ret) {
3887	BT_INSN(insn, "(alt)");
3888	return ret;
3889	}
3890	}
3891
3892	if (skip_orig)
3893	return 0;
3894	}
3895
3896	switch (insn->type) {
3897
3898	case INSN_CALL_DYNAMIC:
3899	case INSN_JUMP_DYNAMIC:
3900	case INSN_JUMP_DYNAMIC_CONDITIONAL:
3901	WARN_INSN(insn, "early indirect call");
3902	return 1;
3903
3904	case INSN_JUMP_UNCONDITIONAL:
3905	case INSN_JUMP_CONDITIONAL:
3906	if (!is_sibling_call(insn)) {
3907	if (!insn->jump_dest) {
3908	WARN_INSN(insn, "unresolved jump target after linking?!?");
3909	return -1;
3910	}
3911	ret = validate_unret(file, insn: insn->jump_dest);
3912	if (ret) {
3913	BT_INSN(insn, "(branch%s)",
3914	insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3915	return ret;
3916	}
3917
3918	if (insn->type == INSN_JUMP_UNCONDITIONAL)
3919	return 0;
3920
3921	break;
3922	}
3923
3924	/* fallthrough */
3925	case INSN_CALL:
3926	dest = find_insn(file, sec: insn_call_dest(insn)->sec,
3927	offset: insn_call_dest(insn)->offset);
3928	if (!dest) {
3929	WARN("Unresolved function after linking!?: %s",
3930	insn_call_dest(insn)->name);
3931	return -1;
3932	}
3933
3934	ret = validate_unret(file, insn: dest);
3935	if (ret) {
3936	BT_INSN(insn, "(call)");
3937	return ret;
3938	}
3939	/*
3940	* If a call returns without error, it must have seen UNTRAIN_RET.
3941	* Therefore any non-error return is a success.
3942	*/
3943	return 0;
3944
3945	case INSN_RETURN:
3946	WARN_INSN(insn, "RET before UNTRAIN");
3947	return 1;
3948
3949	case INSN_NOP:
3950	if (insn->retpoline_safe)
3951	return 0;
3952	break;
3953
3954	default:
3955	break;
3956	}
3957
3958	if (!next) {
3959	WARN_INSN(insn, "teh end!");
3960	return -1;
3961	}
3962	insn = next;
3963	}
3964
3965	return 0;
3966	}
3967
3968	/*
3969	* Validate that all branches starting at VALIDATE_UNRET_BEGIN encounter
3970	* VALIDATE_UNRET_END before RET.
3971	*/
3972	static int validate_unrets(struct objtool_file *file)
3973	{
3974	struct instruction *insn;
3975	int ret, warnings = 0;
3976
3977	for_each_insn(file, insn) {
3978	if (!insn->unret)
3979	continue;
3980
3981	ret = validate_unret(file, insn);
3982	if (ret < 0) {
3983	WARN_INSN(insn, "Failed UNRET validation");
3984	return ret;
3985	}
3986	warnings += ret;
3987	}
3988
3989	return warnings;
3990	}
3991
3992	static int validate_retpoline(struct objtool_file *file)
3993	{
3994	struct instruction *insn;
3995	int warnings = 0;
3996
3997	for_each_insn(file, insn) {
3998	if (insn->type != INSN_JUMP_DYNAMIC &&
3999	insn->type != INSN_CALL_DYNAMIC &&
4000	insn->type != INSN_RETURN)
4001	continue;
4002
4003	if (insn->retpoline_safe)
4004	continue;
4005
4006	if (insn->sec->init)
4007	continue;
4008
4009	if (insn->type == INSN_RETURN) {
4010	if (opts.rethunk) {
4011	WARN_INSN(insn, "'naked' return found in MITIGATION_RETHUNK build");
4012	} else
4013	continue;
4014	} else {
4015	WARN_INSN(insn, "indirect %s found in MITIGATION_RETPOLINE build",
4016	insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
4017	}
4018
4019	warnings++;
4020	}
4021
4022	return warnings;
4023	}
4024
4025	static bool is_kasan_insn(struct instruction *insn)
4026	{
4027	return (insn->type == INSN_CALL &&
4028	!strcmp(insn_call_dest(insn)->name, "__asan_handle_no_return"));
4029	}
4030
4031	static bool is_ubsan_insn(struct instruction *insn)
4032	{
4033	return (insn->type == INSN_CALL &&
4034	!strcmp(insn_call_dest(insn)->name,
4035	"__ubsan_handle_builtin_unreachable"));
4036	}
4037
4038	static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
4039	{
4040	int i;
4041	struct instruction *prev_insn;
4042
4043	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
4044	return true;
4045
4046	/*
4047	* Ignore alternative replacement instructions. This can happen
4048	* when a whitelisted function uses one of the ALTERNATIVE macros.
4049	*/
4050	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
4051	!strcmp(insn->sec->name, ".altinstr_aux"))
4052	return true;
4053
4054	/*
4055	* Whole archive runs might encounter dead code from weak symbols.
4056	* This is where the linker will have dropped the weak symbol in
4057	* favour of a regular symbol, but leaves the code in place.
4058	*
4059	* In this case we'll find a piece of code (whole function) that is not
4060	* covered by a !section symbol. Ignore them.
4061	*/
4062	if (opts.link && !insn_func(insn)) {
4063	int size = find_symbol_hole_containing(insn->sec, insn->offset);
4064	unsigned long end = insn->offset + size;
4065
4066	if (!size) /* not a hole */
4067	return false;
4068
4069	if (size < 0) /* hole until the end */
4070	return true;
4071
4072	sec_for_each_insn_continue(file, insn) {
4073	/*
4074	* If we reach a visited instruction at or before the
4075	* end of the hole, ignore the unreachable.
4076	*/
4077	if (insn->visited)
4078	return true;
4079
4080	if (insn->offset >= end)
4081	break;
4082
4083	/*
4084	* If this hole jumps to a .cold function, mark it ignore too.
4085	*/
4086	if (insn->jump_dest && insn_func(insn->jump_dest) &&
4087	strstr(insn_func(insn->jump_dest)->name, ".cold")) {
4088	struct instruction *dest = insn->jump_dest;
4089	func_for_each_insn(file, insn_func(dest), dest)
4090	dest->ignore = true;
4091	}
4092	}
4093
4094	return false;
4095	}
4096
4097	if (!insn_func(insn))
4098	return false;
4099
4100	if (insn_func(insn)->static_call_tramp)
4101	return true;
4102
4103	/*
4104	* CONFIG_UBSAN_TRAP inserts a UD2 when it sees
4105	* __builtin_unreachable(). The BUG() macro has an unreachable() after
4106	* the UD2, which causes GCC's undefined trap logic to emit another UD2
4107	* (or occasionally a JMP to UD2).
4108	*
4109	* It may also insert a UD2 after calling a __noreturn function.
4110	*/
4111	prev_insn = prev_insn_same_sec(file, insn);
4112	if (prev_insn->dead_end &&
4113	(insn->type == INSN_BUG ||
4114	(insn->type == INSN_JUMP_UNCONDITIONAL &&
4115	insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
4116	return true;
4117
4118	/*
4119	* Check if this (or a subsequent) instruction is related to
4120	* CONFIG_UBSAN or CONFIG_KASAN.
4121	*
4122	* End the search at 5 instructions to avoid going into the weeds.
4123	*/
4124	for (i = 0; i < 5; i++) {
4125
4126	if (is_kasan_insn(insn) || is_ubsan_insn(insn))
4127	return true;
4128
4129	if (insn->type == INSN_JUMP_UNCONDITIONAL) {
4130	if (insn->jump_dest &&
4131	insn_func(insn->jump_dest) == insn_func(insn)) {
4132	insn = insn->jump_dest;
4133	continue;
4134	}
4135
4136	break;
4137	}
4138
4139	if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
4140	break;
4141
4142	insn = next_insn_same_sec(file, insn);
4143	}
4144
4145	return false;
4146	}
4147
4148	static int add_prefix_symbol(struct objtool_file *file, struct symbol *func)
4149	{
4150	struct instruction *insn, *prev;
4151	struct cfi_state *cfi;
4152
4153	insn = find_insn(file, sec: func->sec, offset: func->offset);
4154	if (!insn)
4155	return -1;
4156
4157	for (prev = prev_insn_same_sec(file, insn);
4158	prev;
4159	prev = prev_insn_same_sec(file, insn: prev)) {
4160	u64 offset;
4161
4162	if (prev->type != INSN_NOP)
4163	return -1;
4164
4165	offset = func->offset - prev->offset;
4166
4167	if (offset > opts.prefix)
4168	return -1;
4169
4170	if (offset < opts.prefix)
4171	continue;
4172
4173	elf_create_prefix_symbol(file->elf, func, opts.prefix);
4174	break;
4175	}
4176
4177	if (!prev)
4178	return -1;
4179
4180	if (!insn->cfi) {
4181	/*
4182	* This can happen if stack validation isn't enabled or the
4183	* function is annotated with STACK_FRAME_NON_STANDARD.
4184	*/
4185	return 0;
4186	}
4187
4188	/* Propagate insn->cfi to the prefix code */
4189	cfi = cfi_hash_find_or_add(cfi: insn->cfi);
4190	for (; prev != insn; prev = next_insn_same_sec(file, insn: prev))
4191	prev->cfi = cfi;
4192
4193	return 0;
4194	}
4195
4196	static int add_prefix_symbols(struct objtool_file *file)
4197	{
4198	struct section *sec;
4199	struct symbol *func;
4200
4201	for_each_sec(file, sec) {
4202	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4203	continue;
4204
4205	sec_for_each_sym(sec, func) {
4206	if (func->type != STT_FUNC)
4207	continue;
4208
4209	add_prefix_symbol(file, func);
4210	}
4211	}
4212
4213	return 0;
4214	}
4215
4216	static int validate_symbol(struct objtool_file *file, struct section *sec,
4217	struct symbol *sym, struct insn_state *state)
4218	{
4219	struct instruction *insn;
4220	int ret;
4221
4222	if (!sym->len) {
4223	WARN("%s() is missing an ELF size annotation", sym->name);
4224	return 1;
4225	}
4226
4227	if (sym->pfunc != sym || sym->alias != sym)
4228	return 0;
4229
4230	insn = find_insn(file, sec, offset: sym->offset);
4231	if (!insn || insn->ignore || insn->visited)
4232	return 0;
4233
4234	state->uaccess = sym->uaccess_safe;
4235
4236	ret = validate_branch(file, insn_func(insn), insn, *state);
4237	if (ret)
4238	BT_INSN(insn, "<=== (sym)");
4239	return ret;
4240	}
4241
4242	static int validate_section(struct objtool_file *file, struct section *sec)
4243	{
4244	struct insn_state state;
4245	struct symbol *func;
4246	int warnings = 0;
4247
4248	sec_for_each_sym(sec, func) {
4249	if (func->type != STT_FUNC)
4250	continue;
4251
4252	init_insn_state(file, state: &state, sec);
4253	set_func_state(&state.cfi);
4254
4255	warnings += validate_symbol(file, sec, sym: func, state: &state);
4256	}
4257
4258	return warnings;
4259	}
4260
4261	static int validate_noinstr_sections(struct objtool_file *file)
4262	{
4263	struct section *sec;
4264	int warnings = 0;
4265
4266	sec = find_section_by_name(file->elf, ".noinstr.text");
4267	if (sec) {
4268	warnings += validate_section(file, sec);
4269	warnings += validate_unwind_hints(file, sec);
4270	}
4271
4272	sec = find_section_by_name(file->elf, ".entry.text");
4273	if (sec) {
4274	warnings += validate_section(file, sec);
4275	warnings += validate_unwind_hints(file, sec);
4276	}
4277
4278	sec = find_section_by_name(file->elf, ".cpuidle.text");
4279	if (sec) {
4280	warnings += validate_section(file, sec);
4281	warnings += validate_unwind_hints(file, sec);
4282	}
4283
4284	return warnings;
4285	}
4286
4287	static int validate_functions(struct objtool_file *file)
4288	{
4289	struct section *sec;
4290	int warnings = 0;
4291
4292	for_each_sec(file, sec) {
4293	if (!(sec->sh.sh_flags & SHF_EXECINSTR))
4294	continue;
4295
4296	warnings += validate_section(file, sec);
4297	}
4298
4299	return warnings;
4300	}
4301
4302	static void mark_endbr_used(struct instruction *insn)
4303	{
4304	if (!list_empty(head: &insn->call_node))
4305	list_del_init(entry: &insn->call_node);
4306	}
4307
4308	static bool noendbr_range(struct objtool_file *file, struct instruction *insn)
4309	{
4310	struct symbol *sym = find_symbol_containing(insn->sec, insn->offset-1);
4311	struct instruction *first;
4312
4313	if (!sym)
4314	return false;
4315
4316	first = find_insn(file, sec: sym->sec, offset: sym->offset);
4317	if (!first)
4318	return false;
4319
4320	if (first->type != INSN_ENDBR && !first->noendbr)
4321	return false;
4322
4323	return insn->offset == sym->offset + sym->len;
4324	}
4325
4326	static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
4327	{
4328	struct instruction *dest;
4329	struct reloc *reloc;
4330	unsigned long off;
4331	int warnings = 0;
4332
4333	/*
4334	* Looking for function pointer load relocations. Ignore
4335	* direct/indirect branches:
4336	*/
4337	switch (insn->type) {
4338	case INSN_CALL:
4339	case INSN_CALL_DYNAMIC:
4340	case INSN_JUMP_CONDITIONAL:
4341	case INSN_JUMP_UNCONDITIONAL:
4342	case INSN_JUMP_DYNAMIC:
4343	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4344	case INSN_RETURN:
4345	case INSN_NOP:
4346	return 0;
4347	default:
4348	break;
4349	}
4350
4351	for (reloc = insn_reloc(file, insn);
4352	reloc;
4353	reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4354	reloc_offset(reloc) + 1,
4355	(insn->offset + insn->len) - (reloc_offset(reloc) + 1))) {
4356
4357	/*
4358	* static_call_update() references the trampoline, which
4359	* doesn't have (or need) ENDBR. Skip warning in that case.
4360	*/
4361	if (reloc->sym->static_call_tramp)
4362	continue;
4363
4364	off = reloc->sym->offset;
4365	if (reloc_type(reloc) == R_X86_64_PC32 ||
4366	reloc_type(reloc) == R_X86_64_PLT32)
4367	off += arch_dest_reloc_offset(reloc_addend(reloc));
4368	else
4369	off += reloc_addend(reloc);
4370
4371	dest = find_insn(file, sec: reloc->sym->sec, offset: off);
4372	if (!dest)
4373	continue;
4374
4375	if (dest->type == INSN_ENDBR) {
4376	mark_endbr_used(insn: dest);
4377	continue;
4378	}
4379
4380	if (insn_func(dest) && insn_func(insn) &&
4381	insn_func(dest)->pfunc == insn_func(insn)->pfunc) {
4382	/*
4383	* Anything from->to self is either _THIS_IP_ or
4384	* IRET-to-self.
4385	*
4386	* There is no sane way to annotate _THIS_IP_ since the
4387	* compiler treats the relocation as a constant and is
4388	* happy to fold in offsets, skewing any annotation we
4389	* do, leading to vast amounts of false-positives.
4390	*
4391	* There's also compiler generated _THIS_IP_ through
4392	* KCOV and such which we have no hope of annotating.
4393	*
4394	* As such, blanket accept self-references without
4395	* issue.
4396	*/
4397	continue;
4398	}
4399
4400	/*
4401	* Accept anything ANNOTATE_NOENDBR.
4402	*/
4403	if (dest->noendbr)
4404	continue;
4405
4406	/*
4407	* Accept if this is the instruction after a symbol
4408	* that is (no)endbr -- typical code-range usage.
4409	*/
4410	if (noendbr_range(file, insn: dest))
4411	continue;
4412
4413	WARN_INSN(insn, "relocation to !ENDBR: %s", offstr(dest->sec, dest->offset));
4414
4415	warnings++;
4416	}
4417
4418	return warnings;
4419	}
4420
4421	static int validate_ibt_data_reloc(struct objtool_file *file,
4422	struct reloc *reloc)
4423	{
4424	struct instruction *dest;
4425
4426	dest = find_insn(file, sec: reloc->sym->sec,
4427	offset: reloc->sym->offset + reloc_addend(reloc));
4428	if (!dest)
4429	return 0;
4430
4431	if (dest->type == INSN_ENDBR) {
4432	mark_endbr_used(insn: dest);
4433	return 0;
4434	}
4435
4436	if (dest->noendbr)
4437	return 0;
4438
4439	WARN_FUNC("data relocation to !ENDBR: %s",
4440	reloc->sec->base, reloc_offset(reloc),
4441	offstr(dest->sec, dest->offset));
4442
4443	return 1;
4444	}
4445
4446	/*
4447	* Validate IBT rules and remove used ENDBR instructions from the seal list.
4448	* Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4449	* NOPs) later, in create_ibt_endbr_seal_sections().
4450	*/
4451	static int validate_ibt(struct objtool_file *file)
4452	{
4453	struct section *sec;
4454	struct reloc *reloc;
4455	struct instruction *insn;
4456	int warnings = 0;
4457
4458	for_each_insn(file, insn)
4459	warnings += validate_ibt_insn(file, insn);
4460
4461	for_each_sec(file, sec) {
4462
4463	/* Already done by validate_ibt_insn() */
4464	if (sec->sh.sh_flags & SHF_EXECINSTR)
4465	continue;
4466
4467	if (!sec->rsec)
4468	continue;
4469
4470	/*
4471	* These sections can reference text addresses, but not with
4472	* the intent to indirect branch to them.
4473	*/
4474	if ((!strncmp(sec->name, ".discard", 8) &&
4475	strcmp(sec->name, ".discard.ibt_endbr_noseal")) ||
4476	!strncmp(sec->name, ".debug", 6) ||
4477	!strcmp(sec->name, ".altinstructions") ||
4478	!strcmp(sec->name, ".ibt_endbr_seal") ||
4479	!strcmp(sec->name, ".orc_unwind_ip") ||
4480	!strcmp(sec->name, ".parainstructions") ||
4481	!strcmp(sec->name, ".retpoline_sites") ||
4482	!strcmp(sec->name, ".smp_locks") ||
4483	!strcmp(sec->name, ".static_call_sites") ||
4484	!strcmp(sec->name, "_error_injection_whitelist") ||
4485	!strcmp(sec->name, "_kprobe_blacklist") ||
4486	!strcmp(sec->name, "__bug_table") ||
4487	!strcmp(sec->name, "__ex_table") ||
4488	!strcmp(sec->name, "__jump_table") ||
4489	!strcmp(sec->name, "__mcount_loc") ||
4490	!strcmp(sec->name, ".kcfi_traps") ||
4491	strstr(sec->name, "__patchable_function_entries"))
4492	continue;
4493
4494	for_each_reloc(sec->rsec, reloc)
4495	warnings += validate_ibt_data_reloc(file, reloc);
4496	}
4497
4498	return warnings;
4499	}
4500
4501	static int validate_sls(struct objtool_file *file)
4502	{
4503	struct instruction *insn, *next_insn;
4504	int warnings = 0;
4505
4506	for_each_insn(file, insn) {
4507	next_insn = next_insn_same_sec(file, insn);
4508
4509	if (insn->retpoline_safe)
4510	continue;
4511
4512	switch (insn->type) {
4513	case INSN_RETURN:
4514	if (!next_insn || next_insn->type != INSN_TRAP) {
4515	WARN_INSN(insn, "missing int3 after ret");
4516	warnings++;
4517	}
4518
4519	break;
4520	case INSN_JUMP_DYNAMIC:
4521	if (!next_insn || next_insn->type != INSN_TRAP) {
4522	WARN_INSN(insn, "missing int3 after indirect jump");
4523	warnings++;
4524	}
4525	break;
4526	default:
4527	break;
4528	}
4529	}
4530
4531	return warnings;
4532	}
4533
4534	static bool ignore_noreturn_call(struct instruction *insn)
4535	{
4536	struct symbol *call_dest = insn_call_dest(insn);
4537
4538	/*
4539	* FIXME: hack, we need a real noreturn solution
4540	*
4541	* Problem is, exc_double_fault() may or may not return, depending on
4542	* whether CONFIG_X86_ESPFIX64 is set. But objtool has no visibility
4543	* to the kernel config.
4544	*
4545	* Other potential ways to fix it:
4546	*
4547	* - have compiler communicate __noreturn functions somehow
4548	* - remove CONFIG_X86_ESPFIX64
4549	* - read the .config file
4550	* - add a cmdline option
4551	* - create a generic objtool annotation format (vs a bunch of custom
4552	* formats) and annotate it
4553	*/
4554	if (!strcmp(call_dest->name, "exc_double_fault")) {
4555	/* prevent further unreachable warnings for the caller */
4556	insn->sym->warned = 1;
4557	return true;
4558	}
4559
4560	return false;
4561	}
4562
4563	static int validate_reachable_instructions(struct objtool_file *file)
4564	{
4565	struct instruction *insn, *prev_insn;
4566	struct symbol *call_dest;
4567	int warnings = 0;
4568
4569	if (file->ignore_unreachables)
4570	return 0;
4571
4572	for_each_insn(file, insn) {
4573	if (insn->visited || ignore_unreachable_insn(file, insn))
4574	continue;
4575
4576	prev_insn = prev_insn_same_sec(file, insn);
4577	if (prev_insn && prev_insn->dead_end) {
4578	call_dest = insn_call_dest(prev_insn);
4579	if (call_dest && !ignore_noreturn_call(prev_insn)) {
4580	WARN_INSN(insn, "%s() is missing a __noreturn annotation",
4581	call_dest->name);
4582	warnings++;
4583	continue;
4584	}
4585	}
4586
4587	WARN_INSN(insn, "unreachable instruction");
4588	warnings++;
4589	}
4590
4591	return warnings;
4592	}
4593
4594	/* 'funcs' is a space-separated list of function names */
4595	static int disas_funcs(const char *funcs)
4596	{
4597	const char *objdump_str, *cross_compile;
4598	int size, ret;
4599	char *cmd;
4600
4601	cross_compile = getenv("CROSS_COMPILE");
4602
4603	objdump_str = "%sobjdump -wdr %s | gawk -M -v _funcs='%s' '"
4604	"BEGIN { split(_funcs, funcs); }"
4605	"/^$/ { func_match = 0; }"
4606	"/<.*>:/ { "
4607	"f = gensub(/.*<(.*)>:/, \"\\\\1\", 1);"
4608	"for (i in funcs) {"
4609	"if (funcs[i] == f) {"
4610	"func_match = 1;"
4611	"base = strtonum(\"0x\" $1);"
4612	"break;"
4613	"}"
4614	"}"
4615	"}"
4616	"{"
4617	"if (func_match) {"
4618	"addr = strtonum(\"0x\" $1);"
4619	"printf(\"%%04x \", addr - base);"
4620	"print;"
4621	"}"
4622	"}' 1>&2";
4623
4624	/* fake snprintf() to calculate the size */
4625	size = snprintf(NULL, 0, objdump_str, cross_compile, objname, funcs) + 1;
4626	if (size <= 0) {
4627	WARN("objdump string size calculation failed");
4628	return -1;
4629	}
4630
4631	cmd = malloc(size);
4632
4633	/* real snprintf() */
4634	snprintf(cmd, size, objdump_str, cross_compile, objname, funcs);
4635	ret = system(cmd);
4636	if (ret) {
4637	WARN("disassembly failed: %d", ret);
4638	return -1;
4639	}
4640
4641	return 0;
4642	}
4643
4644	static int disas_warned_funcs(struct objtool_file *file)
4645	{
4646	struct symbol *sym;
4647	char *funcs = NULL, *tmp;
4648
4649	for_each_sym(file, sym) {
4650	if (sym->warned) {
4651	if (!funcs) {
4652	funcs = malloc(strlen(sym->name) + 1);
4653	strcpy(funcs, sym->name);
4654	} else {
4655	tmp = malloc(strlen(funcs) + strlen(sym->name) + 2);
4656	sprintf(buf: tmp, fmt: "%s %s", funcs, sym->name);
4657	free(funcs);
4658	funcs = tmp;
4659	}
4660	}
4661	}
4662
4663	if (funcs)
4664	disas_funcs(funcs);
4665
4666	return 0;
4667	}
4668
4669	struct insn_chunk {
4670	void *addr;
4671	struct insn_chunk *next;
4672	};
4673
4674	/*
4675	* Reduce peak RSS usage by freeing insns memory before writing the ELF file,
4676	* which can trigger more allocations for .debug_* sections whose data hasn't
4677	* been read yet.
4678	*/
4679	static void free_insns(struct objtool_file *file)
4680	{
4681	struct instruction *insn;
4682	struct insn_chunk *chunks = NULL, *chunk;
4683
4684	for_each_insn(file, insn) {
4685	if (!insn->idx) {
4686	chunk = malloc(sizeof(*chunk));
4687	chunk->addr = insn;
4688	chunk->next = chunks;
4689	chunks = chunk;
4690	}
4691	}
4692
4693	for (chunk = chunks; chunk; chunk = chunk->next)
4694	free(chunk->addr);
4695	}
4696
4697	int check(struct objtool_file *file)
4698	{
4699	int ret, warnings = 0;
4700
4701	arch_initial_func_cfi_state(&initial_func_cfi);
4702	init_cfi_state(cfi: &init_cfi);
4703	init_cfi_state(cfi: &func_cfi);
4704	set_func_state(&func_cfi);
4705	init_cfi_state(cfi: &force_undefined_cfi);
4706	force_undefined_cfi.force_undefined = true;
4707
4708	if (!cfi_hash_alloc(size: 1UL << (file->elf->symbol_bits - 3)))
4709	goto out;
4710
4711	cfi_hash_add(cfi: &init_cfi);
4712	cfi_hash_add(cfi: &func_cfi);
4713
4714	ret = decode_sections(file);
4715	if (ret < 0)
4716	goto out;
4717
4718	warnings += ret;
4719
4720	if (!nr_insns)
4721	goto out;
4722
4723	if (opts.retpoline) {
4724	ret = validate_retpoline(file);
4725	if (ret < 0)
4726	return ret;
4727	warnings += ret;
4728	}
4729
4730	if (opts.stackval || opts.orc || opts.uaccess) {
4731	ret = validate_functions(file);
4732	if (ret < 0)
4733	goto out;
4734	warnings += ret;
4735
4736	ret = validate_unwind_hints(file, NULL);
4737	if (ret < 0)
4738	goto out;
4739	warnings += ret;
4740
4741	if (!warnings) {
4742	ret = validate_reachable_instructions(file);
4743	if (ret < 0)
4744	goto out;
4745	warnings += ret;
4746	}
4747
4748	} else if (opts.noinstr) {
4749	ret = validate_noinstr_sections(file);
4750	if (ret < 0)
4751	goto out;
4752	warnings += ret;
4753	}
4754
4755	if (opts.unret) {
4756	/*
4757	* Must be after validate_branch() and friends, it plays
4758	* further games with insn->visited.
4759	*/
4760	ret = validate_unrets(file);
4761	if (ret < 0)
4762	return ret;
4763	warnings += ret;
4764	}
4765
4766	if (opts.ibt) {
4767	ret = validate_ibt(file);
4768	if (ret < 0)
4769	goto out;
4770	warnings += ret;
4771	}
4772
4773	if (opts.sls) {
4774	ret = validate_sls(file);
4775	if (ret < 0)
4776	goto out;
4777	warnings += ret;
4778	}
4779
4780	if (opts.static_call) {
4781	ret = create_static_call_sections(file);
4782	if (ret < 0)
4783	goto out;
4784	warnings += ret;
4785	}
4786
4787	if (opts.retpoline) {
4788	ret = create_retpoline_sites_sections(file);
4789	if (ret < 0)
4790	goto out;
4791	warnings += ret;
4792	}
4793
4794	if (opts.cfi) {
4795	ret = create_cfi_sections(file);
4796	if (ret < 0)
4797	goto out;
4798	warnings += ret;
4799	}
4800
4801	if (opts.rethunk) {
4802	ret = create_return_sites_sections(file);
4803	if (ret < 0)
4804	goto out;
4805	warnings += ret;
4806
4807	if (opts.hack_skylake) {
4808	ret = create_direct_call_sections(file);
4809	if (ret < 0)
4810	goto out;
4811	warnings += ret;
4812	}
4813	}
4814
4815	if (opts.mcount) {
4816	ret = create_mcount_loc_sections(file);
4817	if (ret < 0)
4818	goto out;
4819	warnings += ret;
4820	}
4821
4822	if (opts.prefix) {
4823	ret = add_prefix_symbols(file);
4824	if (ret < 0)
4825	return ret;
4826	warnings += ret;
4827	}
4828
4829	if (opts.ibt) {
4830	ret = create_ibt_endbr_seal_sections(file);
4831	if (ret < 0)
4832	goto out;
4833	warnings += ret;
4834	}
4835
4836	if (opts.orc && nr_insns) {
4837	ret = orc_create(file);
4838	if (ret < 0)
4839	goto out;
4840	warnings += ret;
4841	}
4842
4843	free_insns(file);
4844
4845	if (opts.verbose)
4846	disas_warned_funcs(file);
4847
4848	if (opts.stats) {
4849	printf("nr_insns_visited: %ld\n", nr_insns_visited);
4850	printf("nr_cfi: %ld\n", nr_cfi);
4851	printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4852	printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4853	}
4854
4855	out:
4856	/*
4857	* For now, don't fail the kernel build on fatal warnings. These
4858	* errors are still fairly common due to the growing matrix of
4859	* supported toolchains and their recent pace of change.
4860	*/
4861	return 0;
4862	}
4863