1//! Support code for rustc's built in unit-test and micro-benchmarking
2//! framework.
3//!
4//! Almost all user code will only be interested in `Bencher` and
5//! `black_box`. All other interactions (such as writing tests and
6//! benchmarks themselves) should be done via the `#[test]` and
7//! `#[bench]` attributes.
8//!
9//! See the [Testing Chapter](../book/ch11-00-testing.html) of the book for more details.
10
11// Currently, not much of this is meant for users. It is intended to
12// support the simplest interface possible for representing and
13// running tests while providing a base that other test frameworks may
14// build off of.
15
16// N.B., this is also specified in this crate's Cargo.toml, but librustc_ast contains logic specific to
17// this crate, which relies on this attribute (rather than the value of `--crate-name` passed by
18// cargo) to detect this crate.
19#![cfg_attr(feature = "asm_black_box", feature(test))]
20#![cfg_attr(feature = "capture", feature(internal_output_capture))]
21
22// Public reexports
23pub use self::bench::{black_box, Bencher};
24pub use self::console::run_tests_console;
25pub use self::options::{ColorConfig, Options, OutputFormat, RunIgnored, ShouldPanic};
26pub use self::types::TestName::*;
27pub use self::types::*;
28pub use self::ColorConfig::*;
29pub use cli::TestOpts;
30
31// Module to be used by rustc to compile tests in libtest
32pub mod test {
33 pub use crate::{
34 assert_test_result,
35 bench::Bencher,
36 cli::{parse_opts, TestOpts},
37 filter_tests,
38 helpers::metrics::{Metric, MetricMap},
39 options::{Options, RunIgnored, RunStrategy, ShouldPanic},
40 run_test, test_main, test_main_static,
41 test_result::{TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk},
42 time::{TestExecTime, TestTimeOptions},
43 types::{
44 DynTestFn, DynTestName, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc,
45 TestDescAndFn, TestName, TestType,
46 },
47 };
48}
49
50use std::{
51 env, io,
52 io::prelude::Write,
53 panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo},
54 process::{self, Command},
55 sync::mpsc::{channel, Sender},
56 sync::{Arc, Mutex},
57 thread,
58 time::{Duration, Instant},
59};
60
61pub mod bench;
62mod cli;
63mod console;
64mod event;
65mod formatters;
66mod helpers;
67mod options;
68pub mod stats;
69mod test_result;
70mod time;
71mod types;
72
73#[cfg(test)]
74mod tests;
75
76use event::{CompletedTest, TestEvent};
77use helpers::concurrency::get_concurrency;
78use helpers::exit_code::get_exit_code;
79use options::{Concurrent, RunStrategy};
80use test_result::*;
81use time::TestExecTime;
82
83// Process exit code to be used to indicate test failures.
84const ERROR_EXIT_CODE: i32 = 101;
85
86const SECONDARY_TEST_INVOKER_VAR: &str = "__RUST_TEST_INVOKE";
87
88// The default console test runner. It accepts the command line
89// arguments and a vector of test_descs.
90pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Options>) {
91 let mut opts = match cli::parse_opts(args) {
92 Some(Ok(o)) => o,
93 Some(Err(msg)) => {
94 eprintln!("error: {}", msg);
95 process::exit(ERROR_EXIT_CODE);
96 }
97 None => return,
98 };
99 if let Some(options) = options {
100 opts.options = options;
101 }
102 if opts.list {
103 if let Err(e) = console::list_tests_console(&opts, tests) {
104 eprintln!("error: io error when listing tests: {:?}", e);
105 process::exit(ERROR_EXIT_CODE);
106 }
107 } else {
108 match console::run_tests_console(&opts, tests) {
109 Ok(true) => {}
110 Ok(false) => process::exit(ERROR_EXIT_CODE),
111 Err(e) => {
112 eprintln!("error: io error when listing tests: {:?}", e);
113 process::exit(ERROR_EXIT_CODE);
114 }
115 }
116 }
117}
118
119/// A variant optimized for invocation with a static test vector.
120/// This will panic (intentionally) when fed any dynamic tests.
121///
122/// This is the entry point for the main function generated by `rustc --test`
123/// when panic=unwind.
124pub fn test_main_static(tests: &[&TestDescAndFn]) {
125 let args: Vec = env::args().collect::<Vec<_>>();
126 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
127 test_main(&args, owned_tests, options:None)
128}
129
130/// A variant optimized for invocation with a static test vector.
131/// This will panic (intentionally) when fed any dynamic tests.
132///
133/// Runs tests in panic=abort mode, which involves spawning subprocesses for
134/// tests.
135///
136/// This is the entry point for the main function generated by `rustc --test`
137/// when panic=abort.
138pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
139 // If we're being run in SpawnedSecondary mode, run the test here. run_test
140 // will then exit the process.
141 if let Ok(name: String) = env::var(SECONDARY_TEST_INVOKER_VAR) {
142 env::remove_var(SECONDARY_TEST_INVOKER_VAR);
143 let test: TestDescAndFn = testsOption
144 .iter()
145 .filter(|test: &&&TestDescAndFn| test.desc.name.as_slice() == name)
146 .map(make_owned_test)
147 .next()
148 .unwrap_or_else(|| panic!("couldn't find a test with the provided name '{}'", name));
149 let TestDescAndFn { desc: TestDesc, testfn: TestFn } = test;
150 let testfn: fn() = match testfn {
151 StaticTestFn(f: fn()) => f,
152 _ => panic!("only static tests are supported"),
153 };
154 run_test_in_spawned_subprocess(desc, testfn:Box::new(testfn));
155 }
156
157 let args: Vec = env::args().collect::<Vec<_>>();
158 let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
159 test_main(&args, owned_tests, options:Some(Options::new().panic_abort(true)))
160}
161
162/// Clones static values for putting into a dynamic vector, which test_main()
163/// needs to hand out ownership of tests to parallel test runners.
164///
165/// This will panic when fed any dynamic tests, because they cannot be cloned.
166fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
167 match test.testfn {
168 StaticTestFn(f: fn()) => TestDescAndFn { testfn: StaticTestFn(f), desc: test.desc.clone() },
169 StaticBenchFn(f: fn(&mut Bencher)) => TestDescAndFn { testfn: StaticBenchFn(f), desc: test.desc.clone() },
170 _ => panic!("non-static tests passed to test::test_main_static"),
171 }
172}
173
174/// Invoked when unit tests terminate. Should panic if the unit
175/// Tests is considered a failure. By default, invokes `report()`
176/// and checks for a `0` result.
177pub trait Termination {
178 fn report(self) -> i32;
179}
180
181impl Termination for () {
182 fn report(self) -> i32 {
183 0
184 }
185}
186
187/// Invoked when unit tests terminate. Should panic if the unit
188/// Tests is considered a failure. By default, invokes `report()`
189/// and checks for a `0` result.
190pub fn assert_test_result<T: Termination>(result: T) {
191 let code: i32 = result.report();
192 assert_eq!(
193 code, 0,
194 "the test returned a termination value with a non-zero status code ({}) \
195 which indicates a failure",
196 code
197 );
198}
199
200pub fn run_tests<F>(
201 opts: &TestOpts,
202 tests: Vec<TestDescAndFn>,
203 mut notify_about_test_event: F,
204) -> io::Result<()>
205where
206 F: FnMut(TestEvent) -> io::Result<()>,
207{
208 use std::collections::{self, HashMap};
209 use std::hash::BuildHasherDefault;
210 use std::sync::mpsc::RecvTimeoutError;
211 // Use a deterministic hasher
212 type TestMap =
213 HashMap<TestDesc, Instant, BuildHasherDefault<collections::hash_map::DefaultHasher>>;
214
215 let tests_len = tests.len();
216
217 let mut filtered_tests = filter_tests(opts, tests);
218 if !opts.bench_benchmarks {
219 filtered_tests = convert_benchmarks_to_tests(filtered_tests);
220 }
221
222 let filtered_tests = {
223 let mut filtered_tests = filtered_tests;
224 for test in filtered_tests.iter_mut() {
225 test.desc.name = test.desc.name.with_padding(test.testfn.padding());
226 }
227
228 filtered_tests
229 };
230
231 let filtered_out = tests_len - filtered_tests.len();
232 let event = TestEvent::TeFilteredOut(filtered_out);
233 notify_about_test_event(event)?;
234
235 let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect();
236
237 let event = TestEvent::TeFiltered(filtered_descs);
238 notify_about_test_event(event)?;
239
240 let (filtered_tests, filtered_benchs): (Vec<_>, _) = filtered_tests
241 .into_iter()
242 .partition(|e| matches!(e.testfn, StaticTestFn(_) | DynTestFn(_)));
243
244 let concurrency = opts.test_threads.unwrap_or_else(get_concurrency);
245
246 let mut remaining = filtered_tests;
247 remaining.reverse();
248 let mut pending = 0;
249
250 let (tx, rx) = channel::<CompletedTest>();
251 let run_strategy = if opts.options.panic_abort && !opts.force_run_in_process {
252 RunStrategy::SpawnPrimary
253 } else {
254 RunStrategy::InProcess
255 };
256
257 let mut running_tests: TestMap = HashMap::default();
258
259 fn get_timed_out_tests(running_tests: &mut TestMap) -> Vec<TestDesc> {
260 let now = Instant::now();
261 let timed_out = running_tests
262 .iter()
263 .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone()) } else { None })
264 .collect();
265 for test in &timed_out {
266 running_tests.remove(test);
267 }
268 timed_out
269 }
270
271 fn calc_timeout(running_tests: &TestMap) -> Option<Duration> {
272 running_tests.values().min().map(|next_timeout| {
273 let now = Instant::now();
274 if *next_timeout >= now { *next_timeout - now } else { Duration::new(0, 0) }
275 })
276 }
277
278 if concurrency == 1 {
279 while !remaining.is_empty() {
280 let test = remaining.pop().unwrap();
281 let event = TestEvent::TeWait(test.desc.clone());
282 notify_about_test_event(event)?;
283 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
284 let completed_test = rx.recv().unwrap();
285
286 let event = TestEvent::TeResult(completed_test);
287 notify_about_test_event(event)?;
288 }
289 } else {
290 while pending > 0 || !remaining.is_empty() {
291 while pending < concurrency && !remaining.is_empty() {
292 let test = remaining.pop().unwrap();
293 let timeout = time::get_default_test_timeout();
294 running_tests.insert(test.desc.clone(), timeout);
295
296 let event = TestEvent::TeWait(test.desc.clone());
297 notify_about_test_event(event)?; //here no pad
298 run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
299 pending += 1;
300 }
301
302 let mut res;
303 loop {
304 if let Some(timeout) = calc_timeout(&running_tests) {
305 res = rx.recv_timeout(timeout);
306 for test in get_timed_out_tests(&mut running_tests) {
307 let event = TestEvent::TeTimeout(test);
308 notify_about_test_event(event)?;
309 }
310
311 match res {
312 Err(RecvTimeoutError::Timeout) => {
313 // Result is not yet ready, continue waiting.
314 }
315 _ => {
316 // We've got a result, stop the loop.
317 break;
318 }
319 }
320 } else {
321 res = rx.recv().map_err(|_| RecvTimeoutError::Disconnected);
322 break;
323 }
324 }
325
326 let completed_test = res.unwrap();
327 running_tests.remove(&completed_test.desc);
328
329 let event = TestEvent::TeResult(completed_test);
330 notify_about_test_event(event)?;
331 pending -= 1;
332 }
333 }
334
335 if opts.bench_benchmarks {
336 // All benchmarks run at the end, in serial.
337 for b in filtered_benchs {
338 let event = TestEvent::TeWait(b.desc.clone());
339 notify_about_test_event(event)?;
340 run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
341 let completed_test = rx.recv().unwrap();
342
343 let event = TestEvent::TeResult(completed_test);
344 notify_about_test_event(event)?;
345 }
346 }
347 Ok(())
348}
349
350pub fn filter_tests(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
351 let mut filtered = tests;
352 let matches_filter = |test: &TestDescAndFn, filter: &str| {
353 let test_name = test.desc.name.as_slice();
354
355 match opts.filter_exact {
356 true => test_name == filter,
357 false => test_name.contains(filter),
358 }
359 };
360
361 // Remove tests that don't match the test filter
362 if !opts.filters.is_empty() {
363 filtered.retain(|test| opts.filters.iter().any(|filter| matches_filter(test, filter)));
364 }
365
366 // Skip tests that match any of the skip filters
367 filtered.retain(|test| !opts.skip.iter().any(|sf| matches_filter(test, sf)));
368
369 // Excludes #[should_panic] tests
370 if opts.exclude_should_panic {
371 filtered.retain(|test| test.desc.should_panic == ShouldPanic::No);
372 }
373
374 // maybe unignore tests
375 match opts.run_ignored {
376 RunIgnored::Yes => {
377 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
378 }
379 RunIgnored::Only => {
380 filtered.retain(|test| test.desc.ignore);
381 filtered.iter_mut().for_each(|test| test.desc.ignore = false);
382 }
383 RunIgnored::No => {}
384 }
385
386 // Sort the tests alphabetically
387 filtered.sort_by(|t1, t2| t1.desc.name.as_slice().cmp(t2.desc.name.as_slice()));
388
389 filtered
390}
391
392pub fn convert_benchmarks_to_tests(tests: Vec<TestDescAndFn>) -> Vec<TestDescAndFn> {
393 // convert benchmarks to tests, if we're not benchmarking them
394 testsimpl Iterator
395 .into_iter()
396 .map(|x: TestDescAndFn| {
397 let testfn: TestFn = match x.testfn {
398 DynBenchFn(bench: Box) => DynTestFn(Box::new(move || {
399 bench::run_once(|b: &mut Bencher| __rust_begin_short_backtrace(|| bench.run(harness:b)))
400 })),
401 StaticBenchFn(benchfn: fn(&mut Bencher)) => DynTestFn(Box::new(move || {
402 bench::run_once(|b: &mut Bencher| __rust_begin_short_backtrace(|| benchfn(b)))
403 })),
404 f: TestFn => f,
405 };
406 TestDescAndFn { desc: x.desc, testfn }
407 })
408 .collect()
409}
410
411pub fn run_test(
412 opts: &TestOpts,
413 force_ignore: bool,
414 test: TestDescAndFn,
415 strategy: RunStrategy,
416 monitor_ch: Sender<CompletedTest>,
417 concurrency: Concurrent,
418) {
419 let TestDescAndFn { desc, testfn } = test;
420
421 // Emscripten can catch panics but other wasm targets cannot
422 let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No
423 && cfg!(target_arch = "wasm32")
424 && !cfg!(target_os = "emscripten");
425
426 if force_ignore || desc.ignore || ignore_because_no_process_support {
427 let message = CompletedTest::new(desc, TrIgnored, None, Vec::new());
428 monitor_ch.send(message).unwrap();
429 return;
430 }
431
432 struct TestRunOpts {
433 pub strategy: RunStrategy,
434 pub nocapture: bool,
435 pub concurrency: Concurrent,
436 pub time: Option<time::TestTimeOptions>,
437 }
438
439 fn run_test_inner(
440 desc: TestDesc,
441 monitor_ch: Sender<CompletedTest>,
442 testfn: Box<dyn FnOnce() + Send>,
443 opts: TestRunOpts,
444 ) {
445 let concurrency = opts.concurrency;
446 let name = desc.name.clone();
447
448 let runtest = move || match opts.strategy {
449 RunStrategy::InProcess => run_test_in_process(
450 desc,
451 opts.nocapture,
452 opts.time.is_some(),
453 testfn,
454 monitor_ch,
455 opts.time,
456 ),
457 RunStrategy::SpawnPrimary => spawn_test_subprocess(
458 desc,
459 opts.nocapture,
460 opts.time.is_some(),
461 monitor_ch,
462 opts.time,
463 ),
464 };
465
466 // If the platform is single-threaded we're just going to run
467 // the test synchronously, regardless of the concurrency
468 // level.
469 let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32");
470 if concurrency == Concurrent::Yes && supports_threads {
471 let cfg = thread::Builder::new().name(name.as_slice().to_owned());
472 cfg.spawn(runtest).unwrap();
473 } else {
474 runtest();
475 }
476 }
477
478 let test_run_opts =
479 TestRunOpts { strategy, nocapture: opts.nocapture, concurrency, time: opts.time_options };
480
481 match testfn {
482 DynBenchFn(bencher) => {
483 // Benchmarks aren't expected to panic, so we run them all in-process.
484 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
485 bencher.run(harness)
486 });
487 }
488 StaticBenchFn(benchfn) => {
489 // Benchmarks aren't expected to panic, so we run them all in-process.
490 crate::bench::benchmark(desc, monitor_ch, opts.nocapture, benchfn);
491 }
492 DynTestFn(f) => {
493 match strategy {
494 RunStrategy::InProcess => (),
495 _ => panic!("Cannot run dynamic test fn out-of-process"),
496 };
497 run_test_inner(
498 desc,
499 monitor_ch,
500 Box::new(move || __rust_begin_short_backtrace(f)),
501 test_run_opts,
502 );
503 }
504 StaticTestFn(f) => run_test_inner(
505 desc,
506 monitor_ch,
507 Box::new(move || __rust_begin_short_backtrace(f)),
508 test_run_opts,
509 ),
510 }
511}
512
513/// Fixed frame used to clean the backtrace with `RUST_BACKTRACE=1`.
514#[inline(never)]
515fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
516 f();
517
518 // prevent this frame from being tail-call optimised away
519 black_box(());
520}
521
522fn run_test_in_process(
523 desc: TestDesc,
524 nocapture: bool,
525 report_time: bool,
526 testfn: Box<dyn FnOnce() + Send>,
527 monitor_ch: Sender<CompletedTest>,
528 time_opts: Option<time::TestTimeOptions>,
529) {
530 // Buffer for capturing standard I/O
531 let data = Arc::new(Mutex::new(Vec::new()));
532
533 if !nocapture {
534 #[cfg(feature = "capture")]
535 io::set_output_capture(Some(data.clone()));
536 }
537
538 let start = if report_time { Some(Instant::now()) } else { None };
539 let result = catch_unwind(AssertUnwindSafe(testfn));
540 let exec_time = start.map(|start| {
541 let duration = start.elapsed();
542 TestExecTime(duration)
543 });
544
545 #[cfg(feature = "capture")]
546 io::set_output_capture(None);
547
548 let test_result = match result {
549 Ok(()) => calc_result(&desc, Ok(()), &time_opts, &exec_time),
550 Err(e) => calc_result(&desc, Err(e.as_ref()), &time_opts, &exec_time),
551 };
552 let stdout = data.lock().unwrap_or_else(|e| e.into_inner()).to_vec();
553 let message = CompletedTest::new(desc, test_result, exec_time, stdout);
554 monitor_ch.send(message).unwrap();
555}
556
557fn spawn_test_subprocess(
558 desc: TestDesc,
559 nocapture: bool,
560 report_time: bool,
561 monitor_ch: Sender<CompletedTest>,
562 time_opts: Option<time::TestTimeOptions>,
563) {
564 let (result, test_output, exec_time) = (|| {
565 let args = env::args().collect::<Vec<_>>();
566 let current_exe = &args[0];
567
568 let mut command = Command::new(current_exe);
569 command.env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice());
570 if nocapture {
571 command.stdout(process::Stdio::inherit());
572 command.stderr(process::Stdio::inherit());
573 }
574
575 let start = if report_time { Some(Instant::now()) } else { None };
576 let output = match command.output() {
577 Ok(out) => out,
578 Err(e) => {
579 let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
580 return (TrFailed, err.into_bytes(), None);
581 }
582 };
583 let exec_time = start.map(|start| {
584 let duration = start.elapsed();
585 TestExecTime(duration)
586 });
587
588 let std::process::Output { stdout, stderr, status } = output;
589 let mut test_output = stdout;
590 formatters::write_stderr_delimiter(&mut test_output, &desc.name);
591 test_output.extend_from_slice(&stderr);
592
593 let result = match (|| -> Result<TestResult, String> {
594 let exit_code = get_exit_code(status)?;
595 Ok(get_result_from_exit_code(&desc, exit_code, &time_opts, &exec_time))
596 })() {
597 Ok(r) => r,
598 Err(e) => {
599 write!(&mut test_output, "Unexpected error: {}", e).unwrap();
600 TrFailed
601 }
602 };
603
604 (result, test_output, exec_time)
605 })();
606
607 let message = CompletedTest::new(desc, result, exec_time, test_output);
608 monitor_ch.send(message).unwrap();
609}
610
611fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box<dyn FnOnce() + Send>) -> ! {
612 let builtin_panic_hook = panic::take_hook();
613 let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
614 let test_result = match panic_info {
615 Some(info) => calc_result(&desc, Err(info.payload()), &None, &None),
616 None => calc_result(&desc, Ok(()), &None, &None),
617 };
618
619 // We don't support serializing TrFailedMsg, so just
620 // print the message out to stderr.
621 if let TrFailedMsg(msg) = &test_result {
622 eprintln!("{}", msg);
623 }
624
625 if let Some(info) = panic_info {
626 builtin_panic_hook(info);
627 }
628
629 if let TrOk = test_result {
630 process::exit(test_result::TR_OK);
631 } else {
632 process::exit(test_result::TR_FAILED);
633 }
634 });
635 let record_result2 = record_result.clone();
636 panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
637 testfn();
638 record_result(None);
639 unreachable!("panic=abort callback should have exited the process")
640}
641