1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * sorttable.h |
4 | * |
5 | * Added ORC unwind tables sort support and other updates: |
6 | * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: |
7 | * Shile Zhang <shile.zhang@linux.alibaba.com> |
8 | * |
9 | * Copyright 2011 - 2012 Cavium, Inc. |
10 | * |
11 | * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: |
12 | * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> |
13 | * |
14 | * Some of this code was taken out of recordmcount.h written by: |
15 | * |
16 | * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. |
17 | * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. |
18 | */ |
19 | |
20 | #undef extable_ent_size |
21 | #undef compare_extable |
22 | #undef get_mcount_loc |
23 | #undef sort_mcount_loc |
24 | #undef elf_mcount_loc |
25 | #undef do_sort |
26 | #undef Elf_Addr |
27 | #undef Elf_Ehdr |
28 | #undef Elf_Shdr |
29 | #undef Elf_Rel |
30 | #undef Elf_Rela |
31 | #undef Elf_Sym |
32 | #undef ELF_R_SYM |
33 | #undef Elf_r_sym |
34 | #undef ELF_R_INFO |
35 | #undef Elf_r_info |
36 | #undef ELF_ST_BIND |
37 | #undef ELF_ST_TYPE |
38 | #undef fn_ELF_R_SYM |
39 | #undef fn_ELF_R_INFO |
40 | #undef uint_t |
41 | #undef _r |
42 | #undef _w |
43 | |
44 | #ifdef SORTTABLE_64 |
45 | # define extable_ent_size 16 |
46 | # define compare_extable compare_extable_64 |
47 | # define get_mcount_loc get_mcount_loc_64 |
48 | # define sort_mcount_loc sort_mcount_loc_64 |
49 | # define elf_mcount_loc elf_mcount_loc_64 |
50 | # define do_sort do_sort_64 |
51 | # define Elf_Addr Elf64_Addr |
52 | # define Elf_Ehdr Elf64_Ehdr |
53 | # define Elf_Shdr Elf64_Shdr |
54 | # define Elf_Rel Elf64_Rel |
55 | # define Elf_Rela Elf64_Rela |
56 | # define Elf_Sym Elf64_Sym |
57 | # define ELF_R_SYM ELF64_R_SYM |
58 | # define Elf_r_sym Elf64_r_sym |
59 | # define ELF_R_INFO ELF64_R_INFO |
60 | # define Elf_r_info Elf64_r_info |
61 | # define ELF_ST_BIND ELF64_ST_BIND |
62 | # define ELF_ST_TYPE ELF64_ST_TYPE |
63 | # define fn_ELF_R_SYM fn_ELF64_R_SYM |
64 | # define fn_ELF_R_INFO fn_ELF64_R_INFO |
65 | # define uint_t uint64_t |
66 | # define _r r8 |
67 | # define _w w8 |
68 | #else |
69 | # define extable_ent_size 8 |
70 | # define compare_extable compare_extable_32 |
71 | # define get_mcount_loc get_mcount_loc_32 |
72 | # define sort_mcount_loc sort_mcount_loc_32 |
73 | # define elf_mcount_loc elf_mcount_loc_32 |
74 | # define do_sort do_sort_32 |
75 | # define Elf_Addr Elf32_Addr |
76 | # define Elf_Ehdr Elf32_Ehdr |
77 | # define Elf_Shdr Elf32_Shdr |
78 | # define Elf_Rel Elf32_Rel |
79 | # define Elf_Rela Elf32_Rela |
80 | # define Elf_Sym Elf32_Sym |
81 | # define ELF_R_SYM ELF32_R_SYM |
82 | # define Elf_r_sym Elf32_r_sym |
83 | # define ELF_R_INFO ELF32_R_INFO |
84 | # define Elf_r_info Elf32_r_info |
85 | # define ELF_ST_BIND ELF32_ST_BIND |
86 | # define ELF_ST_TYPE ELF32_ST_TYPE |
87 | # define fn_ELF_R_SYM fn_ELF32_R_SYM |
88 | # define fn_ELF_R_INFO fn_ELF32_R_INFO |
89 | # define uint_t uint32_t |
90 | # define _r r |
91 | # define _w w |
92 | #endif |
93 | |
94 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
95 | /* ORC unwinder only support X86_64 */ |
96 | #include <asm/orc_types.h> |
97 | |
98 | #define ERRSTR_MAXSZ 256 |
99 | |
100 | char g_err[ERRSTR_MAXSZ]; |
101 | int *g_orc_ip_table; |
102 | struct orc_entry *g_orc_table; |
103 | |
104 | pthread_t orc_sort_thread; |
105 | |
106 | static inline unsigned long orc_ip(const int *ip) |
107 | { |
108 | return (unsigned long)ip + *ip; |
109 | } |
110 | |
111 | static int orc_sort_cmp(const void *_a, const void *_b) |
112 | { |
113 | struct orc_entry *orc_a; |
114 | const int *a = g_orc_ip_table + *(int *)_a; |
115 | const int *b = g_orc_ip_table + *(int *)_b; |
116 | unsigned long a_val = orc_ip(a); |
117 | unsigned long b_val = orc_ip(b); |
118 | |
119 | if (a_val > b_val) |
120 | return 1; |
121 | if (a_val < b_val) |
122 | return -1; |
123 | |
124 | /* |
125 | * The "weak" section terminator entries need to always be on the left |
126 | * to ensure the lookup code skips them in favor of real entries. |
127 | * These terminator entries exist to handle any gaps created by |
128 | * whitelisted .o files which didn't get objtool generation. |
129 | */ |
130 | orc_a = g_orc_table + (a - g_orc_ip_table); |
131 | return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; |
132 | } |
133 | |
134 | static void *sort_orctable(void *arg) |
135 | { |
136 | int i; |
137 | int *idxs = NULL; |
138 | int *tmp_orc_ip_table = NULL; |
139 | struct orc_entry *tmp_orc_table = NULL; |
140 | unsigned int *orc_ip_size = (unsigned int *)arg; |
141 | unsigned int num_entries = *orc_ip_size / sizeof(int); |
142 | unsigned int orc_size = num_entries * sizeof(struct orc_entry); |
143 | |
144 | idxs = (int *)malloc(*orc_ip_size); |
145 | if (!idxs) { |
146 | snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s" , |
147 | strerror(errno)); |
148 | pthread_exit(g_err); |
149 | } |
150 | |
151 | tmp_orc_ip_table = (int *)malloc(*orc_ip_size); |
152 | if (!tmp_orc_ip_table) { |
153 | snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s" , |
154 | strerror(errno)); |
155 | pthread_exit(g_err); |
156 | } |
157 | |
158 | tmp_orc_table = (struct orc_entry *)malloc(orc_size); |
159 | if (!tmp_orc_table) { |
160 | snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s" , |
161 | strerror(errno)); |
162 | pthread_exit(g_err); |
163 | } |
164 | |
165 | /* initialize indices array, convert ip_table to absolute address */ |
166 | for (i = 0; i < num_entries; i++) { |
167 | idxs[i] = i; |
168 | tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); |
169 | } |
170 | memcpy(tmp_orc_table, g_orc_table, orc_size); |
171 | |
172 | qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); |
173 | |
174 | for (i = 0; i < num_entries; i++) { |
175 | if (idxs[i] == i) |
176 | continue; |
177 | |
178 | /* convert back to relative address */ |
179 | g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); |
180 | g_orc_table[i] = tmp_orc_table[idxs[i]]; |
181 | } |
182 | |
183 | free(idxs); |
184 | free(tmp_orc_ip_table); |
185 | free(tmp_orc_table); |
186 | pthread_exit(NULL); |
187 | } |
188 | #endif |
189 | |
190 | static int compare_extable(const void *a, const void *b) |
191 | { |
192 | Elf_Addr av = _r(a); |
193 | Elf_Addr bv = _r(b); |
194 | |
195 | if (av < bv) |
196 | return -1; |
197 | if (av > bv) |
198 | return 1; |
199 | return 0; |
200 | } |
201 | #ifdef MCOUNT_SORT_ENABLED |
202 | pthread_t mcount_sort_thread; |
203 | |
204 | struct elf_mcount_loc { |
205 | Elf_Ehdr *ehdr; |
206 | Elf_Shdr *init_data_sec; |
207 | uint_t start_mcount_loc; |
208 | uint_t stop_mcount_loc; |
209 | }; |
210 | |
211 | /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ |
212 | static void *sort_mcount_loc(void *arg) |
213 | { |
214 | struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; |
215 | uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) |
216 | + _r(&(emloc->init_data_sec)->sh_offset); |
217 | uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; |
218 | unsigned char *start_loc = (void *)emloc->ehdr + offset; |
219 | |
220 | qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); |
221 | return NULL; |
222 | } |
223 | |
224 | /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ |
225 | static void get_mcount_loc(uint_t *_start, uint_t *_stop) |
226 | { |
227 | FILE *file_start, *file_stop; |
228 | char start_buff[20]; |
229 | char stop_buff[20]; |
230 | int len = 0; |
231 | |
232 | file_start = popen(" grep start_mcount System.map | awk '{print $1}' " , "r" ); |
233 | if (!file_start) { |
234 | fprintf(stderr, "get start_mcount_loc error!" ); |
235 | return; |
236 | } |
237 | |
238 | file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' " , "r" ); |
239 | if (!file_stop) { |
240 | fprintf(stderr, "get stop_mcount_loc error!" ); |
241 | pclose(file_start); |
242 | return; |
243 | } |
244 | |
245 | while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { |
246 | len = strlen(start_buff); |
247 | start_buff[len - 1] = '\0'; |
248 | } |
249 | *_start = strtoul(start_buff, NULL, 16); |
250 | |
251 | while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { |
252 | len = strlen(stop_buff); |
253 | stop_buff[len - 1] = '\0'; |
254 | } |
255 | *_stop = strtoul(stop_buff, NULL, 16); |
256 | |
257 | pclose(file_start); |
258 | pclose(file_stop); |
259 | } |
260 | #endif |
261 | static int do_sort(Elf_Ehdr *ehdr, |
262 | char const *const fname, |
263 | table_sort_t custom_sort) |
264 | { |
265 | int rc = -1; |
266 | Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); |
267 | Elf_Shdr *strtab_sec = NULL; |
268 | Elf_Shdr *symtab_sec = NULL; |
269 | Elf_Shdr *extab_sec = NULL; |
270 | Elf_Sym *sym; |
271 | const Elf_Sym *symtab; |
272 | Elf32_Word *symtab_shndx = NULL; |
273 | Elf_Sym *sort_needed_sym = NULL; |
274 | Elf_Shdr *sort_needed_sec; |
275 | Elf_Rel *relocs = NULL; |
276 | int relocs_size = 0; |
277 | uint32_t *sort_needed_loc; |
278 | const char *secstrings; |
279 | const char *strtab; |
280 | char *extab_image; |
281 | int extab_index = 0; |
282 | int i; |
283 | int idx; |
284 | unsigned int shnum; |
285 | unsigned int shstrndx; |
286 | #ifdef MCOUNT_SORT_ENABLED |
287 | struct elf_mcount_loc mstruct = {0}; |
288 | uint_t _start_mcount_loc = 0; |
289 | uint_t _stop_mcount_loc = 0; |
290 | #endif |
291 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
292 | unsigned int orc_ip_size = 0; |
293 | unsigned int orc_size = 0; |
294 | unsigned int orc_num_entries = 0; |
295 | #endif |
296 | |
297 | shstrndx = r2(&ehdr->e_shstrndx); |
298 | if (shstrndx == SHN_XINDEX) |
299 | shstrndx = r(&shdr[0].sh_link); |
300 | secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); |
301 | |
302 | shnum = r2(&ehdr->e_shnum); |
303 | if (shnum == SHN_UNDEF) |
304 | shnum = _r(&shdr[0].sh_size); |
305 | |
306 | for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { |
307 | idx = r(&s->sh_name); |
308 | if (!strcmp(s1: secstrings + idx, s2: "__ex_table" )) { |
309 | extab_sec = s; |
310 | extab_index = i; |
311 | } |
312 | if (!strcmp(s1: secstrings + idx, s2: ".symtab" )) |
313 | symtab_sec = s; |
314 | if (!strcmp(s1: secstrings + idx, s2: ".strtab" )) |
315 | strtab_sec = s; |
316 | |
317 | if ((r(&s->sh_type) == SHT_REL || |
318 | r(&s->sh_type) == SHT_RELA) && |
319 | r(&s->sh_info) == extab_index) { |
320 | relocs = (void *)ehdr + _r(&s->sh_offset); |
321 | relocs_size = _r(&s->sh_size); |
322 | } |
323 | if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) |
324 | symtab_shndx = (Elf32_Word *)((const char *)ehdr + |
325 | _r(&s->sh_offset)); |
326 | |
327 | #ifdef MCOUNT_SORT_ENABLED |
328 | /* locate the .init.data section in vmlinux */ |
329 | if (!strcmp(secstrings + idx, ".init.data" )) { |
330 | get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); |
331 | mstruct.ehdr = ehdr; |
332 | mstruct.init_data_sec = s; |
333 | mstruct.start_mcount_loc = _start_mcount_loc; |
334 | mstruct.stop_mcount_loc = _stop_mcount_loc; |
335 | } |
336 | #endif |
337 | |
338 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
339 | /* locate the ORC unwind tables */ |
340 | if (!strcmp(secstrings + idx, ".orc_unwind_ip" )) { |
341 | orc_ip_size = s->sh_size; |
342 | g_orc_ip_table = (int *)((void *)ehdr + |
343 | s->sh_offset); |
344 | } |
345 | if (!strcmp(secstrings + idx, ".orc_unwind" )) { |
346 | orc_size = s->sh_size; |
347 | g_orc_table = (struct orc_entry *)((void *)ehdr + |
348 | s->sh_offset); |
349 | } |
350 | #endif |
351 | } /* for loop */ |
352 | |
353 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
354 | if (!g_orc_ip_table || !g_orc_table) { |
355 | fprintf(stderr, |
356 | "incomplete ORC unwind tables in file: %s\n" , fname); |
357 | goto out; |
358 | } |
359 | |
360 | orc_num_entries = orc_ip_size / sizeof(int); |
361 | if (orc_ip_size % sizeof(int) != 0 || |
362 | orc_size % sizeof(struct orc_entry) != 0 || |
363 | orc_num_entries != orc_size / sizeof(struct orc_entry)) { |
364 | fprintf(stderr, |
365 | "inconsistent ORC unwind table entries in file: %s\n" , |
366 | fname); |
367 | goto out; |
368 | } |
369 | |
370 | /* create thread to sort ORC unwind tables concurrently */ |
371 | if (pthread_create(&orc_sort_thread, NULL, |
372 | sort_orctable, &orc_ip_size)) { |
373 | fprintf(stderr, |
374 | "pthread_create orc_sort_thread failed '%s': %s\n" , |
375 | strerror(errno), fname); |
376 | goto out; |
377 | } |
378 | #endif |
379 | |
380 | #ifdef MCOUNT_SORT_ENABLED |
381 | if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { |
382 | fprintf(stderr, |
383 | "incomplete mcount's sort in file: %s\n" , |
384 | fname); |
385 | goto out; |
386 | } |
387 | |
388 | /* create thread to sort mcount_loc concurrently */ |
389 | if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { |
390 | fprintf(stderr, |
391 | "pthread_create mcount_sort_thread failed '%s': %s\n" , |
392 | strerror(errno), fname); |
393 | goto out; |
394 | } |
395 | #endif |
396 | if (!extab_sec) { |
397 | fprintf(stderr, format: "no __ex_table in file: %s\n" , fname); |
398 | goto out; |
399 | } |
400 | |
401 | if (!symtab_sec) { |
402 | fprintf(stderr, format: "no .symtab in file: %s\n" , fname); |
403 | goto out; |
404 | } |
405 | |
406 | if (!strtab_sec) { |
407 | fprintf(stderr, format: "no .strtab in file: %s\n" , fname); |
408 | goto out; |
409 | } |
410 | |
411 | extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); |
412 | strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); |
413 | symtab = (const Elf_Sym *)((const char *)ehdr + |
414 | _r(&symtab_sec->sh_offset)); |
415 | |
416 | if (custom_sort) { |
417 | custom_sort(extab_image, _r(&extab_sec->sh_size)); |
418 | } else { |
419 | int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; |
420 | qsort(base: extab_image, nmemb: num_entries, |
421 | extable_ent_size, compare_extable); |
422 | } |
423 | |
424 | /* If there were relocations, we no longer need them. */ |
425 | if (relocs) |
426 | memset(s: relocs, c: 0, n: relocs_size); |
427 | |
428 | /* find the flag main_extable_sort_needed */ |
429 | for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); |
430 | sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); |
431 | sym++) { |
432 | if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) |
433 | continue; |
434 | if (!strcmp(s1: strtab + r(&sym->st_name), |
435 | s2: "main_extable_sort_needed" )) { |
436 | sort_needed_sym = sym; |
437 | break; |
438 | } |
439 | } |
440 | |
441 | if (!sort_needed_sym) { |
442 | fprintf(stderr, |
443 | format: "no main_extable_sort_needed symbol in file: %s\n" , |
444 | fname); |
445 | goto out; |
446 | } |
447 | |
448 | sort_needed_sec = &shdr[get_secindex(shndx: r2(&sym->st_shndx), |
449 | sym_offs: sort_needed_sym - symtab, |
450 | symtab_shndx_start: symtab_shndx)]; |
451 | sort_needed_loc = (void *)ehdr + |
452 | _r(&sort_needed_sec->sh_offset) + |
453 | _r(&sort_needed_sym->st_value) - |
454 | _r(&sort_needed_sec->sh_addr); |
455 | |
456 | /* extable has been sorted, clear the flag */ |
457 | w(0, sort_needed_loc); |
458 | rc = 0; |
459 | |
460 | out: |
461 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
462 | if (orc_sort_thread) { |
463 | void *retval = NULL; |
464 | /* wait for ORC tables sort done */ |
465 | rc = pthread_join(orc_sort_thread, &retval); |
466 | if (rc) { |
467 | fprintf(stderr, |
468 | "pthread_join failed '%s': %s\n" , |
469 | strerror(errno), fname); |
470 | } else if (retval) { |
471 | rc = -1; |
472 | fprintf(stderr, |
473 | "failed to sort ORC tables '%s': %s\n" , |
474 | (char *)retval, fname); |
475 | } |
476 | } |
477 | #endif |
478 | |
479 | #ifdef MCOUNT_SORT_ENABLED |
480 | if (mcount_sort_thread) { |
481 | void *retval = NULL; |
482 | /* wait for mcount sort done */ |
483 | rc = pthread_join(mcount_sort_thread, &retval); |
484 | if (rc) { |
485 | fprintf(stderr, |
486 | "pthread_join failed '%s': %s\n" , |
487 | strerror(errno), fname); |
488 | } else if (retval) { |
489 | rc = -1; |
490 | fprintf(stderr, |
491 | "failed to sort mcount '%s': %s\n" , |
492 | (char *)retval, fname); |
493 | } |
494 | } |
495 | #endif |
496 | return rc; |
497 | } |
498 | |