1//! Module converting command-line arguments into test configuration.
2
3use std::env;
4use std::io::{self, IsTerminal, Write};
5use std::path::PathBuf;
6
7use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
8use super::time::TestTimeOptions;
9
10#[derive(Debug)]
11pub struct TestOpts {
12 pub list: bool,
13 pub filters: Vec<String>,
14 pub filter_exact: bool,
15 pub force_run_in_process: bool,
16 pub exclude_should_panic: bool,
17 pub run_ignored: RunIgnored,
18 pub run_tests: bool,
19 pub bench_benchmarks: bool,
20 pub logfile: Option<PathBuf>,
21 pub nocapture: bool,
22 pub color: ColorConfig,
23 pub format: OutputFormat,
24 pub shuffle: bool,
25 pub shuffle_seed: Option<u64>,
26 pub test_threads: Option<usize>,
27 pub skip: Vec<String>,
28 pub time_options: Option<TestTimeOptions>,
29 /// Stop at first failing test.
30 /// May run a few more tests due to threading, but will
31 /// abort as soon as possible.
32 pub fail_fast: bool,
33 pub options: Options,
34}
35
36impl TestOpts {
37 pub fn use_color(&self) -> bool {
38 match self.color {
39 ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
40 ColorConfig::AlwaysColor => true,
41 ColorConfig::NeverColor => false,
42 }
43 }
44}
45
46/// Result of parsing the options.
47pub(crate) type OptRes = Result<TestOpts, String>;
48/// Result of parsing the option part.
49type OptPartRes<T> = Result<T, String>;
50
51fn optgroups() -> getopts::Options {
52 let mut opts = getopts::Options::new();
53 opts.optflag("", "include-ignored", "Run ignored and not ignored tests")
54 .optflag("", "ignored", "Run only ignored tests")
55 .optflag("", "force-run-in-process", "Forces tests to run in-process when panic=abort")
56 .optflag("", "exclude-should-panic", "Excludes tests marked as should_panic")
57 .optflag("", "test", "Run tests and not benchmarks")
58 .optflag("", "bench", "Run benchmarks instead of tests")
59 .optflag("", "list", "List all tests and benchmarks")
60 .optflag("h", "help", "Display this message")
61 .optopt("", "logfile", "Write logs to the specified file (deprecated)", "PATH")
62 .optflag(
63 "",
64 "no-capture",
65 "don't capture stdout/stderr of each \
66 task, allow printing directly",
67 )
68 .optopt(
69 "",
70 "test-threads",
71 "Number of threads used for running tests \
72 in parallel",
73 "n_threads",
74 )
75 .optmulti(
76 "",
77 "skip",
78 "Skip tests whose names contain FILTER (this flag can \
79 be used multiple times)",
80 "FILTER",
81 )
82 .optflag(
83 "q",
84 "quiet",
85 "Display one character per test instead of one line. \
86 Alias to --format=terse",
87 )
88 .optflag("", "exact", "Exactly match filters rather than by substring")
89 .optopt(
90 "",
91 "color",
92 "Configure coloring of output:
93 auto = colorize if stdout is a tty and tests are run on serially (default);
94 always = always colorize output;
95 never = never colorize output;",
96 "auto|always|never",
97 )
98 .optopt(
99 "",
100 "format",
101 "Configure formatting of output:
102 pretty = Print verbose output;
103 terse = Display one character per test;
104 json = Output a json document;
105 junit = Output a JUnit document",
106 "pretty|terse|json|junit",
107 )
108 .optflag("", "show-output", "Show captured stdout of successful tests")
109 .optopt(
110 "Z",
111 "",
112 "Enable nightly-only flags:
113 unstable-options = Allow use of experimental features",
114 "unstable-options",
115 )
116 .optflag(
117 "",
118 "report-time",
119 "Show execution time of each test.
120
121 Threshold values for colorized output can be configured via
122 `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
123 `RUST_TEST_TIME_DOCTEST` environment variables.
124
125 Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
126 Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time
127 is 0.5 seconds, and the critical time is 2 seconds.
128
129 Not available for --format=terse",
130 )
131 .optflag(
132 "",
133 "ensure-time",
134 "Treat excess of the test execution time limit as error.
135
136 Threshold values for this option can be configured via
137 `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and
138 `RUST_TEST_TIME_DOCTEST` environment variables.
139
140 Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`.
141
142 `CRITICAL_TIME` here means the limit that should not be exceeded by test.
143 ",
144 )
145 .optflag("", "shuffle", "Run tests in random order")
146 .optopt(
147 "",
148 "shuffle-seed",
149 "Run tests in random order; seed the random number generator with SEED",
150 "SEED",
151 );
152 opts
153}
154
155fn usage(binary: &str, options: &getopts::Options) {
156 let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]");
157 println!(
158 r#"{usage}
159
160The FILTER string is tested against the name of all tests, and only those
161tests whose names contain the filter are run. Multiple filter strings may
162be passed, which will run all tests matching any of the filters.
163
164By default, all tests are run in parallel. This can be altered with the
165--test-threads flag or the RUST_TEST_THREADS environment variable when running
166tests (set it to 1).
167
168By default, the tests are run in alphabetical order. Use --shuffle or set
169RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated
170"shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the
171tests in the same order again. Note that --shuffle and --shuffle-seed do not
172affect whether the tests are run in parallel.
173
174All tests have their standard output and standard error captured by default.
175This can be overridden with the --no-capture flag or setting RUST_TEST_NOCAPTURE
176environment variable to a value other than "0". Logging is not captured by default.
177
178Test Attributes:
179
180 `#[test]` - Indicates a function is a test to be run. This function
181 takes no arguments.
182 `#[bench]` - Indicates a function is a benchmark to be run. This
183 function takes one argument (test::Bencher).
184 `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if
185 the code causes a panic (an assertion failure or panic!)
186 A message may be provided, which the failure string must
187 contain: #[should_panic(expected = "foo")].
188 `#[ignore]` - When applied to a function which is already attributed as a
189 test, then the test runner will ignore these tests during
190 normal test runs. Running with --ignored or --include-ignored will run
191 these tests."#,
192 usage = options.usage(&message)
193 );
194}
195
196/// Parses command line arguments into test options.
197/// Returns `None` if help was requested (since we only show help message and don't run tests),
198/// returns `Some(Err(..))` if provided arguments are incorrect,
199/// otherwise creates a `TestOpts` object and returns it.
200pub fn parse_opts(args: &[String]) -> Option<OptRes> {
201 // Parse matches.
202 let mut opts = optgroups();
203 // Flags hidden from `usage`
204 opts.optflag("", "nocapture", "Deprecated, use `--no-capture`");
205
206 let binary = args.first().map(|c| &**c).unwrap_or("...");
207 let args = args.get(1..).unwrap_or(args);
208 let matches = match opts.parse(args) {
209 Ok(m) => m,
210 Err(f) => return Some(Err(f.to_string())),
211 };
212
213 // Check if help was requested.
214 if matches.opt_present("h") {
215 // Show help and do nothing more.
216 usage(binary, &optgroups());
217 return None;
218 }
219
220 // Actually parse the opts.
221 let opts_result = parse_opts_impl(matches);
222
223 Some(opts_result)
224}
225
226// Gets the option value and checks if unstable features are enabled.
227macro_rules! unstable_optflag {
228 ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
229 let opt = $matches.opt_present($option_name);
230 if !$allow_unstable && opt {
231 return Err(format!(
232 "The \"{}\" flag is only accepted on the nightly compiler with -Z unstable-options",
233 $option_name
234 ));
235 }
236
237 opt
238 }};
239}
240
241// Gets the option value and checks if unstable features are enabled.
242macro_rules! unstable_optopt {
243 ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{
244 let opt = $matches.opt_str($option_name);
245 if !$allow_unstable && opt.is_some() {
246 return Err(format!(
247 "The \"{}\" option is only accepted on the nightly compiler with -Z unstable-options",
248 $option_name
249 ));
250 }
251
252 opt
253 }};
254}
255
256// Implementation of `parse_opts` that doesn't care about help message
257// and returns a `Result`.
258fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
259 let allow_unstable = get_allow_unstable(&matches)?;
260
261 // Unstable flags
262 let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process");
263 let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic");
264 let time_options = get_time_options(&matches, allow_unstable)?;
265 let shuffle = get_shuffle(&matches, allow_unstable)?;
266 let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?;
267
268 let include_ignored = matches.opt_present("include-ignored");
269 let quiet = matches.opt_present("quiet");
270 let exact = matches.opt_present("exact");
271 let list = matches.opt_present("list");
272 let skip = matches.opt_strs("skip");
273
274 let bench_benchmarks = matches.opt_present("bench");
275 let run_tests = !bench_benchmarks || matches.opt_present("test");
276
277 let logfile = get_log_file(&matches)?;
278 let run_ignored = get_run_ignored(&matches, include_ignored)?;
279 let filters = matches.free.clone();
280 let nocapture = get_nocapture(&matches)?;
281 let test_threads = get_test_threads(&matches)?;
282 let color = get_color_config(&matches)?;
283 let format = get_format(&matches, quiet, allow_unstable)?;
284
285 let options = Options::new().display_output(matches.opt_present("show-output"));
286
287 if logfile.is_some() {
288 let _ = write!(io::stderr(), "warning: `--logfile` is deprecated");
289 }
290
291 let test_opts = TestOpts {
292 list,
293 filters,
294 filter_exact: exact,
295 force_run_in_process,
296 exclude_should_panic,
297 run_ignored,
298 run_tests,
299 bench_benchmarks,
300 logfile,
301 nocapture,
302 color,
303 format,
304 shuffle,
305 shuffle_seed,
306 test_threads,
307 skip,
308 time_options,
309 options,
310 fail_fast: false,
311 };
312
313 Ok(test_opts)
314}
315
316// FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566
317fn is_nightly() -> bool {
318 // Whether this is a feature-staged build, i.e., on the beta or stable channel
319 let disable_unstable_features: bool =
320 option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(default:false);
321 // Whether we should enable unstable features for bootstrapping
322 let bootstrap: bool = env::var(key:"RUSTC_BOOTSTRAP").is_ok();
323
324 bootstrap || !disable_unstable_features
325}
326
327// Gets the CLI options associated with `report-time` feature.
328fn get_time_options(
329 matches: &getopts::Matches,
330 allow_unstable: bool,
331) -> OptPartRes<Option<TestTimeOptions>> {
332 let report_time: bool = unstable_optflag!(matches, allow_unstable, "report-time");
333 let ensure_test_time: bool = unstable_optflag!(matches, allow_unstable, "ensure-time");
334
335 // If `ensure-test-time` option is provided, time output is enforced,
336 // so user won't be confused if any of tests will silently fail.
337 let options: Option = if report_time || ensure_test_time {
338 Some(TestTimeOptions::new_from_env(error_on_excess:ensure_test_time))
339 } else {
340 None
341 };
342
343 Ok(options)
344}
345
346fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> {
347 let mut shuffle: bool = unstable_optflag!(matches, allow_unstable, "shuffle");
348 if !shuffle && allow_unstable {
349 shuffle = match env::var(key:"RUST_TEST_SHUFFLE") {
350 Ok(val: String) => &val != "0",
351 Err(_) => false,
352 };
353 }
354
355 Ok(shuffle)
356}
357
358fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> {
359 let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed") {
360 Some(n_str) => match n_str.parse::<u64>() {
361 Ok(n) => Some(n),
362 Err(e) => {
363 return Err(format!(
364 "argument for --shuffle-seed must be a number \
365 (error: {e})"
366 ));
367 }
368 },
369 None => None,
370 };
371
372 if shuffle_seed.is_none() && allow_unstable {
373 shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED") {
374 Ok(val) => match val.parse::<u64>() {
375 Ok(n) => Some(n),
376 Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number."),
377 },
378 Err(_) => None,
379 };
380 }
381
382 Ok(shuffle_seed)
383}
384
385fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> {
386 let test_threads: Option = match matches.opt_str(nm:"test-threads") {
387 Some(n_str: String) => match n_str.parse::<usize>() {
388 Ok(0) => return Err("argument for --test-threads must not be 0".to_string()),
389 Ok(n: usize) => Some(n),
390 Err(e: ParseIntError) => {
391 return Err(format!(
392 "argument for --test-threads must be a number > 0 \
393 (error: {e})"
394 ));
395 }
396 },
397 None => None,
398 };
399
400 Ok(test_threads)
401}
402
403fn get_format(
404 matches: &getopts::Matches,
405 quiet: bool,
406 allow_unstable: bool,
407) -> OptPartRes<OutputFormat> {
408 let format = match matches.opt_str("format").as_deref() {
409 None if quiet => OutputFormat::Terse,
410 Some("pretty") | None => OutputFormat::Pretty,
411 Some("terse") => OutputFormat::Terse,
412 Some("json") => {
413 if !allow_unstable {
414 return Err("The \"json\" format is only accepted on the nightly compiler with -Z unstable-options".into());
415 }
416 OutputFormat::Json
417 }
418 Some("junit") => {
419 if !allow_unstable {
420 return Err("The \"junit\" format is only accepted on the nightly compiler with -Z unstable-options".into());
421 }
422 OutputFormat::Junit
423 }
424 Some(v) => {
425 return Err(format!(
426 "argument for --format must be pretty, terse, json or junit (was \
427 {v})"
428 ));
429 }
430 };
431
432 Ok(format)
433}
434
435fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> {
436 let color: ColorConfig = match matches.opt_str(nm:"color").as_deref() {
437 Some("auto") | None => ColorConfig::AutoColor,
438 Some("always") => ColorConfig::AlwaysColor,
439 Some("never") => ColorConfig::NeverColor,
440
441 Some(v: &str) => {
442 return Err(format!(
443 "argument for --color must be auto, always, or never (was \
444 {v})"
445 ));
446 }
447 };
448
449 Ok(color)
450}
451
452fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> {
453 let mut nocapture: bool = matches.opt_present(nm:"nocapture") || matches.opt_present(nm:"no-capture");
454 if !nocapture {
455 nocapture = match env::var(key:"RUST_TEST_NOCAPTURE") {
456 Ok(val: String) => &val != "0",
457 Err(_) => false,
458 };
459 }
460
461 Ok(nocapture)
462}
463
464fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> {
465 let run_ignored: RunIgnored = match (include_ignored, matches.opt_present(nm:"ignored")) {
466 (true, true) => {
467 return Err("the options --include-ignored and --ignored are mutually exclusive".into());
468 }
469 (true, false) => RunIgnored::Yes,
470 (false, true) => RunIgnored::Only,
471 (false, false) => RunIgnored::No,
472 };
473
474 Ok(run_ignored)
475}
476
477fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> {
478 let mut allow_unstable: bool = false;
479
480 if let Some(opt: String) = matches.opt_str(nm:"Z") {
481 if !is_nightly() {
482 return Err("the option `Z` is only accepted on the nightly compiler".into());
483 }
484
485 match &*opt {
486 "unstable-options" => {
487 allow_unstable = true;
488 }
489 _ => {
490 return Err("Unrecognized option to `Z`".into());
491 }
492 }
493 };
494
495 Ok(allow_unstable)
496}
497
498fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
499 let logfile: Option = matches.opt_str(nm:"logfile").map(|s: String| PathBuf::from(&s));
500
501 Ok(logfile)
502}
503