1//===-- dfsan.cpp ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file is a part of DataFlowSanitizer.
10//
11// DataFlowSanitizer runtime. This file defines the public interface to
12// DataFlowSanitizer as well as the definition of certain runtime functions
13// called automatically by the compiler (specifically the instrumentation pass
14// in llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp).
15//
16// The public interface is defined in include/sanitizer/dfsan_interface.h whose
17// functions are prefixed dfsan_ while the compiler interface functions are
18// prefixed __dfsan_.
19//===----------------------------------------------------------------------===//
20
21#include "dfsan/dfsan.h"
22
23#include "dfsan/dfsan_chained_origin_depot.h"
24#include "dfsan/dfsan_flags.h"
25#include "dfsan/dfsan_origin.h"
26#include "dfsan/dfsan_thread.h"
27#include "sanitizer_common/sanitizer_atomic.h"
28#include "sanitizer_common/sanitizer_common.h"
29#include "sanitizer_common/sanitizer_file.h"
30#include "sanitizer_common/sanitizer_flag_parser.h"
31#include "sanitizer_common/sanitizer_flags.h"
32#include "sanitizer_common/sanitizer_internal_defs.h"
33#include "sanitizer_common/sanitizer_libc.h"
34#include "sanitizer_common/sanitizer_report_decorator.h"
35#include "sanitizer_common/sanitizer_stacktrace.h"
36#if SANITIZER_LINUX
37# include <sys/personality.h>
38#endif
39
40using namespace __dfsan;
41
42Flags __dfsan::flags_data;
43
44// The size of TLS variables. These constants must be kept in sync with the ones
45// in DataFlowSanitizer.cpp.
46static const int kDFsanArgTlsSize = 800;
47static const int kDFsanRetvalTlsSize = 800;
48static const int kDFsanArgOriginTlsSize = 800;
49
50SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
51 __dfsan_retval_tls[kDFsanRetvalTlsSize / sizeof(u64)];
52SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32 __dfsan_retval_origin_tls;
53SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
54 __dfsan_arg_tls[kDFsanArgTlsSize / sizeof(u64)];
55SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32
56 __dfsan_arg_origin_tls[kDFsanArgOriginTlsSize / sizeof(u32)];
57
58// Instrumented code may set this value in terms of -dfsan-track-origins.
59// * undefined or 0: do not track origins.
60// * 1: track origins at memory store operations.
61// * 2: track origins at memory load and store operations.
62// TODO: track callsites.
63extern "C" SANITIZER_WEAK_ATTRIBUTE const int __dfsan_track_origins;
64
65extern "C" SANITIZER_INTERFACE_ATTRIBUTE int dfsan_get_track_origins() {
66 return &__dfsan_track_origins ? __dfsan_track_origins : 0;
67}
68
69// On Linux/x86_64, memory is laid out as follows:
70//
71// +--------------------+ 0x800000000000 (top of memory)
72// | application 3 |
73// +--------------------+ 0x700000000000
74// | invalid |
75// +--------------------+ 0x610000000000
76// | origin 1 |
77// +--------------------+ 0x600000000000
78// | application 2 |
79// +--------------------+ 0x510000000000
80// | shadow 1 |
81// +--------------------+ 0x500000000000
82// | invalid |
83// +--------------------+ 0x400000000000
84// | origin 3 |
85// +--------------------+ 0x300000000000
86// | shadow 3 |
87// +--------------------+ 0x200000000000
88// | origin 2 |
89// +--------------------+ 0x110000000000
90// | invalid |
91// +--------------------+ 0x100000000000
92// | shadow 2 |
93// +--------------------+ 0x010000000000
94// | application 1 |
95// +--------------------+ 0x000000000000
96//
97// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
98// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
99
100extern "C" SANITIZER_INTERFACE_ATTRIBUTE
101dfsan_label __dfsan_union_load(const dfsan_label *ls, uptr n) {
102 dfsan_label label = ls[0];
103 for (uptr i = 1; i != n; ++i)
104 label |= ls[i];
105 return label;
106}
107
108// Return the union of all the n labels from addr at the high 32 bit, and the
109// origin of the first taint byte at the low 32 bit.
110extern "C" SANITIZER_INTERFACE_ATTRIBUTE u64
111__dfsan_load_label_and_origin(const void *addr, uptr n) {
112 dfsan_label label = 0;
113 u64 ret = 0;
114 uptr p = (uptr)addr;
115 dfsan_label *s = shadow_for(ptr: (void *)p);
116 for (uptr i = 0; i < n; ++i) {
117 dfsan_label l = s[i];
118 if (!l)
119 continue;
120 label |= l;
121 if (!ret)
122 ret = *(dfsan_origin *)origin_for(ptr: (void *)(p + i));
123 }
124 return ret | (u64)label << 32;
125}
126
127extern "C" SANITIZER_INTERFACE_ATTRIBUTE
128void __dfsan_unimplemented(char *fname) {
129 if (flags().warn_unimplemented)
130 Report(format: "WARNING: DataFlowSanitizer: call to uninstrumented function %s\n",
131 fname);
132}
133
134extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_wrapper_extern_weak_null(
135 const void *addr, char *fname) {
136 if (!addr)
137 Report(
138 format: "ERROR: DataFlowSanitizer: dfsan generated wrapper calling null "
139 "extern_weak function %s\nIf this only happens with dfsan, the "
140 "dfsan instrumentation pass may be accidentally optimizing out a "
141 "null check\n",
142 fname);
143}
144
145// Use '-mllvm -dfsan-debug-nonzero-labels' and break on this function
146// to try to figure out where labels are being introduced in a nominally
147// label-free program.
148extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_nonzero_label() {
149 if (flags().warn_nonzero_labels)
150 Report(format: "WARNING: DataFlowSanitizer: saw nonzero label\n");
151}
152
153// Indirect call to an uninstrumented vararg function. We don't have a way of
154// handling these at the moment.
155extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
156__dfsan_vararg_wrapper(const char *fname) {
157 Report(format: "FATAL: DataFlowSanitizer: unsupported indirect call to vararg "
158 "function %s\n", fname);
159 Die();
160}
161
162// Resolves the union of two labels.
163SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
164dfsan_union(dfsan_label l1, dfsan_label l2) {
165 return l1 | l2;
166}
167
168static const uptr kOriginAlign = sizeof(dfsan_origin);
169static const uptr kOriginAlignMask = ~(kOriginAlign - 1UL);
170
171static uptr OriginAlignUp(uptr u) {
172 return (u + kOriginAlign - 1) & kOriginAlignMask;
173}
174
175static uptr OriginAlignDown(uptr u) { return u & kOriginAlignMask; }
176
177// Return the origin of the first taint byte in the size bytes from the address
178// addr.
179static dfsan_origin GetOriginIfTainted(uptr addr, uptr size) {
180 for (uptr i = 0; i < size; ++i, ++addr) {
181 dfsan_label *s = shadow_for(ptr: (void *)addr);
182
183 if (*s) {
184 // Validate address region.
185 CHECK(MEM_IS_SHADOW(s));
186 return *(dfsan_origin *)origin_for(ptr: (void *)addr);
187 }
188 }
189 return 0;
190}
191
192// For platforms which support slow unwinder only, we need to restrict the store
193// context size to 1, basically only storing the current pc, because the slow
194// unwinder which is based on libunwind is not async signal safe and causes
195// random freezes in forking applications as well as in signal handlers.
196// DFSan supports only Linux. So we do not restrict the store context size.
197#define GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
198 BufferedStackTrace stack; \
199 stack.Unwind(pc, bp, nullptr, true, flags().store_context_size);
200
201#define PRINT_CALLER_STACK_TRACE \
202 { \
203 GET_CALLER_PC_BP; \
204 GET_STORE_STACK_TRACE_PC_BP(pc, bp) \
205 stack.Print(); \
206 }
207
208// Return a chain with the previous ID id and the current stack.
209// from_init = true if this is the first chain of an origin tracking path.
210static u32 ChainOrigin(u32 id, StackTrace *stack, bool from_init = false) {
211 // StackDepot is not async signal safe. Do not create new chains in a signal
212 // handler.
213 DFsanThread *t = GetCurrentThread();
214 if (t && t->InSignalHandler())
215 return id;
216
217 // As an optimization the origin of an application byte is updated only when
218 // its shadow is non-zero. Because we are only interested in the origins of
219 // taint labels, it does not matter what origin a zero label has. This reduces
220 // memory write cost. MSan does similar optimization. The following invariant
221 // may not hold because of some bugs. We check the invariant to help debug.
222 if (!from_init && id == 0 && flags().check_origin_invariant) {
223 Printf(format: " DFSan found invalid origin invariant\n");
224 PRINT_CALLER_STACK_TRACE
225 }
226
227 Origin o = Origin::FromRawId(id);
228 stack->tag = StackTrace::TAG_UNKNOWN;
229 Origin chained = Origin::CreateChainedOrigin(prev: o, stack);
230 return chained.raw_id();
231}
232
233static void ChainAndWriteOriginIfTainted(uptr src, uptr size, uptr dst,
234 StackTrace *stack) {
235 dfsan_origin o = GetOriginIfTainted(addr: src, size);
236 if (o) {
237 o = ChainOrigin(id: o, stack);
238 *(dfsan_origin *)origin_for(ptr: (void *)dst) = o;
239 }
240}
241
242// Copy the origins of the size bytes from src to dst. The source and target
243// memory ranges cannot be overlapped. This is used by memcpy. stack records the
244// stack trace of the memcpy. When dst and src are not 4-byte aligned properly,
245// origins at the unaligned address boundaries may be overwritten because four
246// contiguous bytes share the same origin.
247static void CopyOrigin(const void *dst, const void *src, uptr size,
248 StackTrace *stack) {
249 uptr d = (uptr)dst;
250 uptr beg = OriginAlignDown(u: d);
251 // Copy left unaligned origin if that memory is tainted.
252 if (beg < d) {
253 ChainAndWriteOriginIfTainted(src: (uptr)src, size: beg + kOriginAlign - d, dst: beg, stack);
254 beg += kOriginAlign;
255 }
256
257 uptr end = OriginAlignDown(u: d + size);
258 // If both ends fall into the same 4-byte slot, we are done.
259 if (end < beg)
260 return;
261
262 // Copy right unaligned origin if that memory is tainted.
263 if (end < d + size)
264 ChainAndWriteOriginIfTainted(src: (uptr)src + (end - d), size: (d + size) - end, dst: end,
265 stack);
266
267 if (beg >= end)
268 return;
269
270 // Align src up.
271 uptr src_a = OriginAlignUp(u: (uptr)src);
272 dfsan_origin *src_o = origin_for(ptr: (void *)src_a);
273 u32 *src_s = (u32 *)shadow_for(ptr: (void *)src_a);
274 dfsan_origin *src_end = origin_for(ptr: (void *)(src_a + (end - beg)));
275 dfsan_origin *dst_o = origin_for(ptr: (void *)beg);
276 dfsan_origin last_src_o = 0;
277 dfsan_origin last_dst_o = 0;
278 for (; src_o < src_end; ++src_o, ++src_s, ++dst_o) {
279 if (!*src_s)
280 continue;
281 if (*src_o != last_src_o) {
282 last_src_o = *src_o;
283 last_dst_o = ChainOrigin(id: last_src_o, stack);
284 }
285 *dst_o = last_dst_o;
286 }
287}
288
289// Copy the origins of the size bytes from src to dst. The source and target
290// memory ranges may be overlapped. So the copy is done in a reverse order.
291// This is used by memmove. stack records the stack trace of the memmove.
292static void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
293 StackTrace *stack) {
294 uptr d = (uptr)dst;
295 uptr end = OriginAlignDown(u: d + size);
296
297 // Copy right unaligned origin if that memory is tainted.
298 if (end < d + size)
299 ChainAndWriteOriginIfTainted(src: (uptr)src + (end - d), size: (d + size) - end, dst: end,
300 stack);
301
302 uptr beg = OriginAlignDown(u: d);
303
304 if (beg + kOriginAlign < end) {
305 // Align src up.
306 uptr src_a = OriginAlignUp(u: (uptr)src);
307 void *src_end = (void *)(src_a + end - beg - kOriginAlign);
308 dfsan_origin *src_end_o = origin_for(ptr: src_end);
309 u32 *src_end_s = (u32 *)shadow_for(ptr: src_end);
310 dfsan_origin *src_begin_o = origin_for(ptr: (void *)src_a);
311 dfsan_origin *dst = origin_for(ptr: (void *)(end - kOriginAlign));
312 dfsan_origin last_src_o = 0;
313 dfsan_origin last_dst_o = 0;
314 for (; src_end_o >= src_begin_o; --src_end_o, --src_end_s, --dst) {
315 if (!*src_end_s)
316 continue;
317 if (*src_end_o != last_src_o) {
318 last_src_o = *src_end_o;
319 last_dst_o = ChainOrigin(id: last_src_o, stack);
320 }
321 *dst = last_dst_o;
322 }
323 }
324
325 // Copy left unaligned origin if that memory is tainted.
326 if (beg < d)
327 ChainAndWriteOriginIfTainted(src: (uptr)src, size: beg + kOriginAlign - d, dst: beg, stack);
328}
329
330// Copy or move the origins of the len bytes from src to dst. The source and
331// target memory ranges may or may not be overlapped. This is used by memory
332// transfer operations. stack records the stack trace of the memory transfer
333// operation.
334static void MoveOrigin(const void *dst, const void *src, uptr size,
335 StackTrace *stack) {
336 // Validate address regions.
337 if (!MEM_IS_SHADOW(shadow_for(dst)) ||
338 !MEM_IS_SHADOW(shadow_for((void *)((uptr)dst + size))) ||
339 !MEM_IS_SHADOW(shadow_for(src)) ||
340 !MEM_IS_SHADOW(shadow_for((void *)((uptr)src + size)))) {
341 CHECK(false);
342 return;
343 }
344 // If destination origin range overlaps with source origin range, move
345 // origins by copying origins in a reverse order; otherwise, copy origins in
346 // a normal order. The orders of origin transfer are consistent with the
347 // orders of how memcpy and memmove transfer user data.
348 uptr src_aligned_beg = OriginAlignDown(u: (uptr)src);
349 uptr src_aligned_end = OriginAlignDown(u: (uptr)src + size);
350 uptr dst_aligned_beg = OriginAlignDown(u: (uptr)dst);
351 if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
352 return ReverseCopyOrigin(dst, src, size, stack);
353 return CopyOrigin(dst, src, size, stack);
354}
355
356// Set the size bytes from the addres dst to be the origin value.
357static void SetOrigin(const void *dst, uptr size, u32 origin) {
358 if (size == 0)
359 return;
360
361 // Origin mapping is 4 bytes per 4 bytes of application memory.
362 // Here we extend the range such that its left and right bounds are both
363 // 4 byte aligned.
364 uptr x = unaligned_origin_for(ptr: (uptr)dst);
365 uptr beg = OriginAlignDown(u: x);
366 uptr end = OriginAlignUp(u: x + size); // align up.
367 u64 origin64 = ((u64)origin << 32) | origin;
368 // This is like memset, but the value is 32-bit. We unroll by 2 to write
369 // 64 bits at once. May want to unroll further to get 128-bit stores.
370 if (beg & 7ULL) {
371 if (*(u32 *)beg != origin)
372 *(u32 *)beg = origin;
373 beg += 4;
374 }
375 for (uptr addr = beg; addr < (end & ~7UL); addr += 8) {
376 if (*(u64 *)addr == origin64)
377 continue;
378 *(u64 *)addr = origin64;
379 }
380 if (end & 7ULL)
381 if (*(u32 *)(end - kOriginAlign) != origin)
382 *(u32 *)(end - kOriginAlign) = origin;
383}
384
385#define RET_CHAIN_ORIGIN(id) \
386 GET_CALLER_PC_BP; \
387 GET_STORE_STACK_TRACE_PC_BP(pc, bp); \
388 return ChainOrigin(id, &stack);
389
390// Return a new origin chain with the previous ID id and the current stack
391// trace.
392extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
393__dfsan_chain_origin(dfsan_origin id) {
394 RET_CHAIN_ORIGIN(id)
395}
396
397// Return a new origin chain with the previous ID id and the current stack
398// trace if the label is tainted.
399extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
400__dfsan_chain_origin_if_tainted(dfsan_label label, dfsan_origin id) {
401 if (!label)
402 return id;
403 RET_CHAIN_ORIGIN(id)
404}
405
406// Copy or move the origins of the len bytes from src to dst.
407extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_mem_origin_transfer(
408 const void *dst, const void *src, uptr len) {
409 if (src == dst)
410 return;
411 GET_CALLER_PC_BP;
412 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
413 MoveOrigin(dst, src, size: len, stack: &stack);
414}
415
416extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_origin_transfer(
417 const void *dst, const void *src, uptr len) {
418 __dfsan_mem_origin_transfer(dst, src, len);
419}
420
421static void CopyShadow(void *dst, const void *src, uptr len) {
422 internal_memcpy(dest: (void *)__dfsan::shadow_for(ptr: dst),
423 src: (const void *)__dfsan::shadow_for(ptr: src),
424 n: len * sizeof(dfsan_label));
425}
426
427extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_mem_shadow_transfer(
428 void *dst, const void *src, uptr len) {
429 CopyShadow(dst, src, len);
430}
431
432// Copy shadow and origins of the len bytes from src to dst.
433extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
434__dfsan_mem_shadow_origin_transfer(void *dst, const void *src, uptr size) {
435 if (src == dst)
436 return;
437 CopyShadow(dst, src, len: size);
438 if (dfsan_get_track_origins()) {
439 // Duplicating code instead of calling __dfsan_mem_origin_transfer
440 // so that the getting the caller stack frame works correctly.
441 GET_CALLER_PC_BP;
442 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
443 MoveOrigin(dst, src, size, stack: &stack);
444 }
445}
446
447// Copy shadow and origins as per __atomic_compare_exchange.
448extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
449__dfsan_mem_shadow_origin_conditional_exchange(u8 condition, void *target,
450 void *expected,
451 const void *desired, uptr size) {
452 void *dst;
453 const void *src;
454 // condition is result of native call to __atomic_compare_exchange
455 if (condition) {
456 // Copy desired into target
457 dst = target;
458 src = desired;
459 } else {
460 // Copy target into expected
461 dst = expected;
462 src = target;
463 }
464 if (src == dst)
465 return;
466 CopyShadow(dst, src, len: size);
467 if (dfsan_get_track_origins()) {
468 // Duplicating code instead of calling __dfsan_mem_origin_transfer
469 // so that the getting the caller stack frame works correctly.
470 GET_CALLER_PC_BP;
471 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
472 MoveOrigin(dst, src, size, stack: &stack);
473 }
474}
475
476namespace __dfsan {
477
478bool dfsan_inited = false;
479bool dfsan_init_is_running = false;
480
481void dfsan_copy_memory(void *dst, const void *src, uptr size) {
482 internal_memcpy(dest: dst, src, n: size);
483 dfsan_mem_shadow_transfer(dst, src, len: size);
484 if (dfsan_get_track_origins())
485 dfsan_mem_origin_transfer(dst, src, len: size);
486}
487
488// Releases the pages within the origin address range.
489static void ReleaseOrigins(void *addr, uptr size) {
490 const uptr beg_origin_addr = (uptr)__dfsan::origin_for(ptr: addr);
491 const void *end_addr = (void *)((uptr)addr + size);
492 const uptr end_origin_addr = (uptr)__dfsan::origin_for(ptr: end_addr);
493
494 if (end_origin_addr - beg_origin_addr <
495 common_flags()->clear_shadow_mmap_threshold)
496 return;
497
498 const uptr page_size = GetPageSizeCached();
499 const uptr beg_aligned = RoundUpTo(size: beg_origin_addr, boundary: page_size);
500 const uptr end_aligned = RoundDownTo(x: end_origin_addr, boundary: page_size);
501
502 if (!MmapFixedSuperNoReserve(fixed_addr: beg_aligned, size: end_aligned - beg_aligned))
503 Die();
504}
505
506static void WriteZeroShadowInRange(uptr beg, uptr end) {
507 // Don't write the label if it is already the value we need it to be.
508 // In a program where most addresses are not labeled, it is common that
509 // a page of shadow memory is entirely zeroed. The Linux copy-on-write
510 // implementation will share all of the zeroed pages, making a copy of a
511 // page when any value is written. The un-sharing will happen even if
512 // the value written does not change the value in memory. Avoiding the
513 // write when both |label| and |*labelp| are zero dramatically reduces
514 // the amount of real memory used by large programs.
515 if (!mem_is_zero(mem: (const char *)beg, size: end - beg))
516 internal_memset(s: (void *)beg, c: 0, n: end - beg);
517}
518
519// Releases the pages within the shadow address range, and sets
520// the shadow addresses not on the pages to be 0.
521static void ReleaseOrClearShadows(void *addr, uptr size) {
522 const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(ptr: addr);
523 const void *end_addr = (void *)((uptr)addr + size);
524 const uptr end_shadow_addr = (uptr)__dfsan::shadow_for(ptr: end_addr);
525
526 if (end_shadow_addr - beg_shadow_addr <
527 common_flags()->clear_shadow_mmap_threshold) {
528 WriteZeroShadowInRange(beg: beg_shadow_addr, end: end_shadow_addr);
529 return;
530 }
531
532 const uptr page_size = GetPageSizeCached();
533 const uptr beg_aligned = RoundUpTo(size: beg_shadow_addr, boundary: page_size);
534 const uptr end_aligned = RoundDownTo(x: end_shadow_addr, boundary: page_size);
535
536 if (beg_aligned >= end_aligned) {
537 WriteZeroShadowInRange(beg: beg_shadow_addr, end: end_shadow_addr);
538 } else {
539 if (beg_aligned != beg_shadow_addr)
540 WriteZeroShadowInRange(beg: beg_shadow_addr, end: beg_aligned);
541 if (end_aligned != end_shadow_addr)
542 WriteZeroShadowInRange(beg: end_aligned, end: end_shadow_addr);
543 if (!MmapFixedSuperNoReserve(fixed_addr: beg_aligned, size: end_aligned - beg_aligned))
544 Die();
545 }
546}
547
548void SetShadow(dfsan_label label, void *addr, uptr size, dfsan_origin origin) {
549 if (0 != label) {
550 const uptr beg_shadow_addr = (uptr)__dfsan::shadow_for(ptr: addr);
551 internal_memset(s: (void *)beg_shadow_addr, c: label, n: size);
552 if (dfsan_get_track_origins())
553 SetOrigin(dst: addr, size, origin);
554 return;
555 }
556
557 if (dfsan_get_track_origins())
558 ReleaseOrigins(addr, size);
559
560 ReleaseOrClearShadows(addr, size);
561}
562
563} // namespace __dfsan
564
565// If the label s is tainted, set the size bytes from the address p to be a new
566// origin chain with the previous ID o and the current stack trace. This is
567// used by instrumentation to reduce code size when too much code is inserted.
568extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_maybe_store_origin(
569 dfsan_label s, void *p, uptr size, dfsan_origin o) {
570 if (UNLIKELY(s)) {
571 GET_CALLER_PC_BP;
572 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
573 SetOrigin(dst: p, size, origin: ChainOrigin(id: o, stack: &stack));
574 }
575}
576
577extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_set_label(
578 dfsan_label label, dfsan_origin origin, void *addr, uptr size) {
579 __dfsan::SetShadow(label, addr, size, origin);
580}
581
582SANITIZER_INTERFACE_ATTRIBUTE
583void dfsan_set_label(dfsan_label label, void *addr, uptr size) {
584 dfsan_origin init_origin = 0;
585 if (label && dfsan_get_track_origins()) {
586 GET_CALLER_PC_BP;
587 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
588 init_origin = ChainOrigin(id: 0, stack: &stack, from_init: true);
589 }
590 __dfsan::SetShadow(label, addr, size, origin: init_origin);
591}
592
593SANITIZER_INTERFACE_ATTRIBUTE
594void dfsan_add_label(dfsan_label label, void *addr, uptr size) {
595 if (0 == label)
596 return;
597
598 if (dfsan_get_track_origins()) {
599 GET_CALLER_PC_BP;
600 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
601 dfsan_origin init_origin = ChainOrigin(id: 0, stack: &stack, from_init: true);
602 SetOrigin(dst: addr, size, origin: init_origin);
603 }
604
605 for (dfsan_label *labelp = shadow_for(ptr: addr); size != 0; --size, ++labelp)
606 *labelp |= label;
607}
608
609// Unlike the other dfsan interface functions the behavior of this function
610// depends on the label of one of its arguments. Hence it is implemented as a
611// custom function.
612extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
613__dfsw_dfsan_get_label(long data, dfsan_label data_label,
614 dfsan_label *ret_label) {
615 *ret_label = 0;
616 return data_label;
617}
618
619extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label __dfso_dfsan_get_label(
620 long data, dfsan_label data_label, dfsan_label *ret_label,
621 dfsan_origin data_origin, dfsan_origin *ret_origin) {
622 *ret_label = 0;
623 *ret_origin = 0;
624 return data_label;
625}
626
627// This function is used if dfsan_get_origin is called when origin tracking is
628// off.
629extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfsw_dfsan_get_origin(
630 long data, dfsan_label data_label, dfsan_label *ret_label) {
631 *ret_label = 0;
632 return 0;
633}
634
635extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin __dfso_dfsan_get_origin(
636 long data, dfsan_label data_label, dfsan_label *ret_label,
637 dfsan_origin data_origin, dfsan_origin *ret_origin) {
638 *ret_label = 0;
639 *ret_origin = 0;
640 return data_origin;
641}
642
643SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
644dfsan_read_label(const void *addr, uptr size) {
645 if (size == 0)
646 return 0;
647 return __dfsan_union_load(ls: shadow_for(ptr: addr), n: size);
648}
649
650SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
651dfsan_read_origin_of_first_taint(const void *addr, uptr size) {
652 return GetOriginIfTainted(addr: (uptr)addr, size);
653}
654
655SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_label_origin(dfsan_label label,
656 dfsan_origin origin,
657 void *addr,
658 uptr size) {
659 __dfsan_set_label(label, origin, addr, size);
660}
661
662extern "C" SANITIZER_INTERFACE_ATTRIBUTE int
663dfsan_has_label(dfsan_label label, dfsan_label elem) {
664 return (label & elem) == elem;
665}
666
667namespace __dfsan {
668
669typedef void (*dfsan_conditional_callback_t)(dfsan_label label,
670 dfsan_origin origin);
671static dfsan_conditional_callback_t conditional_callback = nullptr;
672static dfsan_label labels_in_signal_conditional = 0;
673
674static void ConditionalCallback(dfsan_label label, dfsan_origin origin) {
675 // Programs have many branches. For efficiency the conditional sink callback
676 // handler needs to ignore as many as possible as early as possible.
677 if (label == 0) {
678 return;
679 }
680 if (conditional_callback == nullptr) {
681 return;
682 }
683
684 // This initial ConditionalCallback handler needs to be in here in dfsan
685 // runtime (rather than being an entirely user implemented hook) so that it
686 // has access to dfsan thread information.
687 DFsanThread *t = GetCurrentThread();
688 // A callback operation which does useful work (like record the flow) will
689 // likely be too long executed in a signal handler.
690 if (t && t->InSignalHandler()) {
691 // Record set of labels used in signal handler for completeness.
692 labels_in_signal_conditional |= label;
693 return;
694 }
695
696 conditional_callback(label, origin);
697}
698
699} // namespace __dfsan
700
701extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
702__dfsan_conditional_callback_origin(dfsan_label label, dfsan_origin origin) {
703 __dfsan::ConditionalCallback(label, origin);
704}
705
706extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __dfsan_conditional_callback(
707 dfsan_label label) {
708 __dfsan::ConditionalCallback(label, origin: 0);
709}
710
711extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_set_conditional_callback(
712 __dfsan::dfsan_conditional_callback_t callback) {
713 __dfsan::conditional_callback = callback;
714}
715
716extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
717dfsan_get_labels_in_signal_conditional() {
718 return __dfsan::labels_in_signal_conditional;
719}
720
721namespace __dfsan {
722
723typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label,
724 dfsan_origin origin,
725 const char *file,
726 unsigned int line,
727 const char *function);
728static dfsan_reaches_function_callback_t reaches_function_callback = nullptr;
729static dfsan_label labels_in_signal_reaches_function = 0;
730
731static void ReachesFunctionCallback(dfsan_label label, dfsan_origin origin,
732 const char *file, unsigned int line,
733 const char *function) {
734 if (label == 0) {
735 return;
736 }
737 if (reaches_function_callback == nullptr) {
738 return;
739 }
740
741 // This initial ReachesFunctionCallback handler needs to be in here in dfsan
742 // runtime (rather than being an entirely user implemented hook) so that it
743 // has access to dfsan thread information.
744 DFsanThread *t = GetCurrentThread();
745 // A callback operation which does useful work (like record the flow) will
746 // likely be too long executed in a signal handler.
747 if (t && t->InSignalHandler()) {
748 // Record set of labels used in signal handler for completeness.
749 labels_in_signal_reaches_function |= label;
750 return;
751 }
752
753 reaches_function_callback(label, origin, file, line, function);
754}
755
756} // namespace __dfsan
757
758extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
759__dfsan_reaches_function_callback_origin(dfsan_label label, dfsan_origin origin,
760 const char *file, unsigned int line,
761 const char *function) {
762 __dfsan::ReachesFunctionCallback(label, origin, file, line, function);
763}
764
765extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
766__dfsan_reaches_function_callback(dfsan_label label, const char *file,
767 unsigned int line, const char *function) {
768 __dfsan::ReachesFunctionCallback(label, origin: 0, file, line, function);
769}
770
771extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
772dfsan_set_reaches_function_callback(
773 __dfsan::dfsan_reaches_function_callback_t callback) {
774 __dfsan::reaches_function_callback = callback;
775}
776
777extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_label
778dfsan_get_labels_in_signal_reaches_function() {
779 return __dfsan::labels_in_signal_reaches_function;
780}
781
782class Decorator : public __sanitizer::SanitizerCommonDecorator {
783 public:
784 Decorator() : SanitizerCommonDecorator() {}
785 const char *Origin() const { return Magenta(); }
786};
787
788namespace {
789
790void PrintNoOriginTrackingWarning() {
791 Decorator d;
792 Printf(
793 format: " %sDFSan: origin tracking is not enabled. Did you specify the "
794 "-dfsan-track-origins=1 option?%s\n",
795 d.Warning(), d.Default());
796}
797
798void PrintNoTaintWarning(const void *address) {
799 Decorator d;
800 Printf(format: " %sDFSan: no tainted value at %x%s\n", d.Warning(), address,
801 d.Default());
802}
803
804void PrintInvalidOriginWarning(dfsan_label label, const void *address) {
805 Decorator d;
806 Printf(
807 format: " %sTaint value 0x%x (at %p) has invalid origin tracking. This can "
808 "be a DFSan bug.%s\n",
809 d.Warning(), label, address, d.Default());
810}
811
812void PrintInvalidOriginIdWarning(dfsan_origin origin) {
813 Decorator d;
814 Printf(
815 format: " %sOrigin Id %d has invalid origin tracking. This can "
816 "be a DFSan bug.%s\n",
817 d.Warning(), origin, d.Default());
818}
819
820bool PrintOriginTraceFramesToStr(Origin o, InternalScopedString *out) {
821 Decorator d;
822 bool found = false;
823
824 while (o.isChainedOrigin()) {
825 StackTrace stack;
826 dfsan_origin origin_id = o.raw_id();
827 o = o.getNextChainedOrigin(stack: &stack);
828 if (o.isChainedOrigin())
829 out->AppendF(
830 format: " %sOrigin value: 0x%x, Taint value was stored to memory at%s\n",
831 d.Origin(), origin_id, d.Default());
832 else
833 out->AppendF(format: " %sOrigin value: 0x%x, Taint value was created at%s\n",
834 d.Origin(), origin_id, d.Default());
835
836 // Includes a trailing newline, so no need to add it again.
837 stack.PrintTo(output: out);
838 found = true;
839 }
840
841 return found;
842}
843
844bool PrintOriginTraceToStr(const void *addr, const char *description,
845 InternalScopedString *out) {
846 CHECK(out);
847 CHECK(dfsan_get_track_origins());
848 Decorator d;
849
850 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
851 CHECK(label);
852
853 const dfsan_origin origin = *__dfsan::origin_for(ptr: addr);
854
855 out->AppendF(format: " %sTaint value 0x%x (at %p) origin tracking (%s)%s\n",
856 d.Origin(), label, addr, description ? description : "",
857 d.Default());
858
859 Origin o = Origin::FromRawId(id: origin);
860 return PrintOriginTraceFramesToStr(o, out);
861}
862
863} // namespace
864
865extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_trace(
866 const void *addr, const char *description) {
867 if (!dfsan_get_track_origins()) {
868 PrintNoOriginTrackingWarning();
869 return;
870 }
871
872 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
873 if (!label) {
874 PrintNoTaintWarning(address: addr);
875 return;
876 }
877
878 InternalScopedString trace;
879 bool success = PrintOriginTraceToStr(addr, description, out: &trace);
880
881 if (trace.length())
882 Printf(format: "%s", trace.data());
883
884 if (!success)
885 PrintInvalidOriginWarning(label, address: addr);
886}
887
888extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
889dfsan_sprint_origin_trace(const void *addr, const char *description,
890 char *out_buf, uptr out_buf_size) {
891 CHECK(out_buf);
892
893 if (!dfsan_get_track_origins()) {
894 PrintNoOriginTrackingWarning();
895 return 0;
896 }
897
898 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
899 if (!label) {
900 PrintNoTaintWarning(address: addr);
901 return 0;
902 }
903
904 InternalScopedString trace;
905 bool success = PrintOriginTraceToStr(addr, description, out: &trace);
906
907 if (!success) {
908 PrintInvalidOriginWarning(label, address: addr);
909 return 0;
910 }
911
912 if (out_buf_size) {
913 internal_strncpy(dst: out_buf, src: trace.data(), n: out_buf_size - 1);
914 out_buf[out_buf_size - 1] = '\0';
915 }
916
917 return trace.length();
918}
919
920extern "C" SANITIZER_INTERFACE_ATTRIBUTE void dfsan_print_origin_id_trace(
921 dfsan_origin origin) {
922 if (!dfsan_get_track_origins()) {
923 PrintNoOriginTrackingWarning();
924 return;
925 }
926 Origin o = Origin::FromRawId(id: origin);
927
928 InternalScopedString trace;
929 bool success = PrintOriginTraceFramesToStr(o, out: &trace);
930
931 if (trace.length())
932 Printf(format: "%s", trace.data());
933
934 if (!success)
935 PrintInvalidOriginIdWarning(origin);
936}
937
938extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr dfsan_sprint_origin_id_trace(
939 dfsan_origin origin, char *out_buf, uptr out_buf_size) {
940 CHECK(out_buf);
941
942 if (!dfsan_get_track_origins()) {
943 PrintNoOriginTrackingWarning();
944 return 0;
945 }
946 Origin o = Origin::FromRawId(id: origin);
947
948 InternalScopedString trace;
949 bool success = PrintOriginTraceFramesToStr(o, out: &trace);
950
951 if (!success) {
952 PrintInvalidOriginIdWarning(origin);
953 return 0;
954 }
955
956 if (out_buf_size) {
957 internal_strncpy(dst: out_buf, src: trace.data(), n: out_buf_size - 1);
958 out_buf[out_buf_size - 1] = '\0';
959 }
960
961 return trace.length();
962}
963
964extern "C" SANITIZER_INTERFACE_ATTRIBUTE dfsan_origin
965dfsan_get_init_origin(const void *addr) {
966 if (!dfsan_get_track_origins())
967 return 0;
968
969 const dfsan_label label = *__dfsan::shadow_for(ptr: addr);
970 if (!label)
971 return 0;
972
973 const dfsan_origin origin = *__dfsan::origin_for(ptr: addr);
974
975 Origin o = Origin::FromRawId(id: origin);
976 dfsan_origin origin_id = o.raw_id();
977 while (o.isChainedOrigin()) {
978 StackTrace stack;
979 origin_id = o.raw_id();
980 o = o.getNextChainedOrigin(stack: &stack);
981 }
982 return origin_id;
983}
984
985void __sanitizer::BufferedStackTrace::UnwindImpl(uptr pc, uptr bp,
986 void *context,
987 bool request_fast,
988 u32 max_depth) {
989 using namespace __dfsan;
990 DFsanThread *t = GetCurrentThread();
991 if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind: request_fast)) {
992 return Unwind(max_depth, pc, bp, context, stack_top: 0, stack_bottom: 0, request_fast_unwind: false);
993 }
994 Unwind(max_depth, pc, bp, context: nullptr, stack_top: t->stack_top(), stack_bottom: t->stack_bottom(), request_fast_unwind: true);
995}
996
997extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_stack_trace() {
998 GET_CALLER_PC_BP;
999 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
1000 stack.Print();
1001}
1002
1003extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr
1004dfsan_sprint_stack_trace(char *out_buf, uptr out_buf_size) {
1005 CHECK(out_buf);
1006 GET_CALLER_PC_BP;
1007 GET_STORE_STACK_TRACE_PC_BP(pc, bp);
1008 return stack.PrintTo(out_buf, out_buf_size);
1009}
1010
1011void Flags::SetDefaults() {
1012#define DFSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
1013#include "dfsan_flags.inc"
1014#undef DFSAN_FLAG
1015}
1016
1017static void RegisterDfsanFlags(FlagParser *parser, Flags *f) {
1018#define DFSAN_FLAG(Type, Name, DefaultValue, Description) \
1019 RegisterFlag(parser, #Name, Description, &f->Name);
1020#include "dfsan_flags.inc"
1021#undef DFSAN_FLAG
1022}
1023
1024static void InitializeFlags() {
1025 SetCommonFlagsDefaults();
1026 {
1027 CommonFlags cf;
1028 cf.CopyFrom(other: *common_flags());
1029 cf.intercept_tls_get_addr = true;
1030 OverrideCommonFlags(cf);
1031 }
1032 flags().SetDefaults();
1033
1034 FlagParser parser;
1035 RegisterCommonFlags(parser: &parser);
1036 RegisterDfsanFlags(parser: &parser, f: &flags());
1037 parser.ParseStringFromEnv(env_name: "DFSAN_OPTIONS");
1038 InitializeCommonFlags();
1039 if (Verbosity()) ReportUnrecognizedFlags();
1040 if (common_flags()->help) parser.PrintFlagDescriptions();
1041}
1042
1043SANITIZER_INTERFACE_ATTRIBUTE
1044void dfsan_clear_arg_tls(uptr offset, uptr size) {
1045 internal_memset(s: (void *)((uptr)__dfsan_arg_tls + offset), c: 0, n: size);
1046}
1047
1048SANITIZER_INTERFACE_ATTRIBUTE
1049void dfsan_clear_thread_local_state() {
1050 internal_memset(s: __dfsan_arg_tls, c: 0, n: sizeof(__dfsan_arg_tls));
1051 internal_memset(s: __dfsan_retval_tls, c: 0, n: sizeof(__dfsan_retval_tls));
1052
1053 if (dfsan_get_track_origins()) {
1054 internal_memset(s: __dfsan_arg_origin_tls, c: 0, n: sizeof(__dfsan_arg_origin_tls));
1055 internal_memset(s: &__dfsan_retval_origin_tls, c: 0,
1056 n: sizeof(__dfsan_retval_origin_tls));
1057 }
1058}
1059
1060SANITIZER_INTERFACE_ATTRIBUTE
1061void dfsan_set_arg_tls(uptr offset, dfsan_label label) {
1062 // 2x to match ShadowTLSAlignment.
1063 // ShadowTLSAlignment should probably be changed.
1064 // TODO: Consider reducing ShadowTLSAlignment to 1.
1065 // Aligning to 2 bytes is probably a remnant of fast16 mode.
1066 ((dfsan_label *)__dfsan_arg_tls)[offset * 2] = label;
1067}
1068
1069SANITIZER_INTERFACE_ATTRIBUTE
1070void dfsan_set_arg_origin_tls(uptr offset, dfsan_origin o) {
1071 __dfsan_arg_origin_tls[offset] = o;
1072}
1073
1074extern "C" void dfsan_flush() {
1075 const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1076 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1077 uptr start = kMemoryLayout[i].start;
1078 uptr end = kMemoryLayout[i].end;
1079 uptr size = end - start;
1080 MappingDesc::Type type = kMemoryLayout[i].type;
1081
1082 if (type != MappingDesc::SHADOW && type != MappingDesc::ORIGIN)
1083 continue;
1084
1085 // Check if the segment should be mapped based on platform constraints.
1086 if (start >= maxVirtualAddress)
1087 continue;
1088
1089 if (!MmapFixedSuperNoReserve(fixed_addr: start, size, name: kMemoryLayout[i].name)) {
1090 Printf(format: "FATAL: DataFlowSanitizer: failed to clear memory region\n");
1091 Die();
1092 }
1093 }
1094 __dfsan::labels_in_signal_conditional = 0;
1095 __dfsan::labels_in_signal_reaches_function = 0;
1096}
1097
1098// TODO: CheckMemoryLayoutSanity is based on msan.
1099// Consider refactoring these into a shared implementation.
1100static void CheckMemoryLayoutSanity() {
1101 uptr prev_end = 0;
1102 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1103 uptr start = kMemoryLayout[i].start;
1104 uptr end = kMemoryLayout[i].end;
1105 MappingDesc::Type type = kMemoryLayout[i].type;
1106 CHECK_LT(start, end);
1107 CHECK_EQ(prev_end, start);
1108 CHECK(addr_is_type(start, type));
1109 CHECK(addr_is_type((start + end) / 2, type));
1110 CHECK(addr_is_type(end - 1, type));
1111 if (type == MappingDesc::APP) {
1112 uptr addr = start;
1113 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1114 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1115 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1116
1117 addr = (start + end) / 2;
1118 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1119 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1120 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1121
1122 addr = end - 1;
1123 CHECK(MEM_IS_SHADOW(MEM_TO_SHADOW(addr)));
1124 CHECK(MEM_IS_ORIGIN(MEM_TO_ORIGIN(addr)));
1125 CHECK_EQ(MEM_TO_ORIGIN(addr), SHADOW_TO_ORIGIN(MEM_TO_SHADOW(addr)));
1126 }
1127 prev_end = end;
1128 }
1129}
1130
1131// TODO: CheckMemoryRangeAvailability is based on msan.
1132// Consider refactoring these into a shared implementation.
1133static bool CheckMemoryRangeAvailability(uptr beg, uptr size, bool verbose) {
1134 if (size > 0) {
1135 uptr end = beg + size - 1;
1136 if (!MemoryRangeIsAvailable(range_start: beg, range_end: end)) {
1137 if (verbose)
1138 Printf(format: "FATAL: Memory range %p - %p is not available.\n", beg, end);
1139 return false;
1140 }
1141 }
1142 return true;
1143}
1144
1145// TODO: ProtectMemoryRange is based on msan.
1146// Consider refactoring these into a shared implementation.
1147static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) {
1148 if (size > 0) {
1149 void *addr = MmapFixedNoAccess(fixed_addr: beg, size, name);
1150 if (beg == 0 && addr) {
1151 // Depending on the kernel configuration, we may not be able to protect
1152 // the page at address zero.
1153 uptr gap = 16 * GetPageSizeCached();
1154 beg += gap;
1155 size -= gap;
1156 addr = MmapFixedNoAccess(fixed_addr: beg, size, name);
1157 }
1158 if ((uptr)addr != beg) {
1159 uptr end = beg + size - 1;
1160 Printf(format: "FATAL: Cannot protect memory range %p - %p (%s).\n", beg, end,
1161 name);
1162 return false;
1163 }
1164 }
1165 return true;
1166}
1167
1168// TODO: InitShadow is based on msan.
1169// Consider refactoring these into a shared implementation.
1170bool InitShadow(bool init_origins, bool dry_run) {
1171 // Let user know mapping parameters first.
1172 VPrintf(1, "dfsan_init %p\n", (void *)&__dfsan::dfsan_init);
1173 for (unsigned i = 0; i < kMemoryLayoutSize; ++i)
1174 VPrintf(1, "%s: %zx - %zx\n", kMemoryLayout[i].name, kMemoryLayout[i].start,
1175 kMemoryLayout[i].end - 1);
1176
1177 CheckMemoryLayoutSanity();
1178
1179 if (!MEM_IS_APP(&__dfsan::dfsan_init)) {
1180 if (!dry_run)
1181 Printf(format: "FATAL: Code %p is out of application range. Non-PIE build?\n",
1182 (uptr)&__dfsan::dfsan_init);
1183 return false;
1184 }
1185
1186 const uptr maxVirtualAddress = GetMaxUserVirtualAddress();
1187
1188 for (unsigned i = 0; i < kMemoryLayoutSize; ++i) {
1189 uptr start = kMemoryLayout[i].start;
1190 uptr end = kMemoryLayout[i].end;
1191 uptr size = end - start;
1192 MappingDesc::Type type = kMemoryLayout[i].type;
1193
1194 // Check if the segment should be mapped based on platform constraints.
1195 if (start >= maxVirtualAddress)
1196 continue;
1197
1198 bool map = type == MappingDesc::SHADOW ||
1199 (init_origins && type == MappingDesc::ORIGIN);
1200 bool protect = type == MappingDesc::INVALID ||
1201 (!init_origins && type == MappingDesc::ORIGIN);
1202 CHECK(!(map && protect));
1203 if (!map && !protect) {
1204 CHECK(type == MappingDesc::APP || type == MappingDesc::ALLOCATOR);
1205
1206 if (dry_run && type == MappingDesc::ALLOCATOR &&
1207 !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1208 return false;
1209 }
1210 if (map) {
1211 if (dry_run && !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1212 return false;
1213 if (!dry_run &&
1214 !MmapFixedSuperNoReserve(fixed_addr: start, size, name: kMemoryLayout[i].name))
1215 return false;
1216 if (!dry_run && common_flags()->use_madv_dontdump)
1217 DontDumpShadowMemory(addr: start, length: size);
1218 }
1219 if (protect) {
1220 if (dry_run && !CheckMemoryRangeAvailability(beg: start, size, verbose: !dry_run))
1221 return false;
1222 if (!dry_run && !ProtectMemoryRange(beg: start, size, name: kMemoryLayout[i].name))
1223 return false;
1224 }
1225 }
1226
1227 return true;
1228}
1229
1230bool InitShadowWithReExec(bool init_origins) {
1231 // Start with dry run: check layout is ok, but don't print warnings because
1232 // warning messages will cause tests to fail (even if we successfully re-exec
1233 // after the warning).
1234 bool success = InitShadow(init_origins, dry_run: true);
1235 if (!success) {
1236#if SANITIZER_LINUX
1237 // Perhaps ASLR entropy is too high. If ASLR is enabled, re-exec without it.
1238 int old_personality = personality(persona: 0xffffffff);
1239 bool aslr_on =
1240 (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0);
1241
1242 if (aslr_on) {
1243 VReport(1,
1244 "WARNING: DataflowSanitizer: memory layout is incompatible, "
1245 "possibly due to high-entropy ASLR.\n"
1246 "Re-execing with fixed virtual address space.\n"
1247 "N.B. reducing ASLR entropy is preferable.\n");
1248 CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1);
1249 ReExec();
1250 }
1251#endif
1252 }
1253
1254 // The earlier dry run didn't actually map or protect anything. Run again in
1255 // non-dry run mode.
1256 return success && InitShadow(init_origins, dry_run: false);
1257}
1258
1259static void DFsanInit(int argc, char **argv, char **envp) {
1260 CHECK(!dfsan_init_is_running);
1261 if (dfsan_inited)
1262 return;
1263 dfsan_init_is_running = true;
1264 SanitizerToolName = "DataflowSanitizer";
1265
1266 AvoidCVE_2016_2143();
1267
1268 InitializeFlags();
1269
1270 CheckASLR();
1271
1272 if (!InitShadowWithReExec(init_origins: dfsan_get_track_origins())) {
1273 Printf(format: "FATAL: DataflowSanitizer can not mmap the shadow memory.\n");
1274 DumpProcessMap();
1275 Die();
1276 }
1277
1278 initialize_interceptors();
1279
1280 // Set up threads
1281 DFsanTSDInit(destructor: DFsanTSDDtor);
1282
1283 dfsan_allocator_init();
1284
1285 DFsanThread *main_thread = DFsanThread::Create(start_routine: nullptr, arg: nullptr);
1286 SetCurrentThread(main_thread);
1287 main_thread->Init();
1288
1289 dfsan_init_is_running = false;
1290 dfsan_inited = true;
1291}
1292
1293namespace __dfsan {
1294
1295void dfsan_init() { DFsanInit(argc: 0, argv: nullptr, envp: nullptr); }
1296
1297} // namespace __dfsan
1298
1299#if SANITIZER_CAN_USE_PREINIT_ARRAY
1300__attribute__((section(".preinit_array"),
1301 used)) static void (*dfsan_init_ptr)(int, char **,
1302 char **) = DFsanInit;
1303#endif
1304

source code of compiler-rt/lib/dfsan/dfsan.cpp