1 | //! Module converting command-line arguments into test configuration. |
2 | |
3 | use std::env; |
4 | use std::path::PathBuf; |
5 | |
6 | use super::options::{ColorConfig, Options, OutputFormat, RunIgnored}; |
7 | use super::time::TestTimeOptions; |
8 | use std::io::{self, IsTerminal}; |
9 | |
10 | #[derive(Debug)] |
11 | pub struct TestOpts { |
12 | pub list: bool, |
13 | pub filters: Vec<String>, |
14 | pub filter_exact: bool, |
15 | pub force_run_in_process: bool, |
16 | pub exclude_should_panic: bool, |
17 | pub run_ignored: RunIgnored, |
18 | pub run_tests: bool, |
19 | pub bench_benchmarks: bool, |
20 | pub logfile: Option<PathBuf>, |
21 | pub nocapture: bool, |
22 | pub color: ColorConfig, |
23 | pub format: OutputFormat, |
24 | pub shuffle: bool, |
25 | pub shuffle_seed: Option<u64>, |
26 | pub test_threads: Option<usize>, |
27 | pub skip: Vec<String>, |
28 | pub time_options: Option<TestTimeOptions>, |
29 | /// Stop at first failing test. |
30 | /// May run a few more tests due to threading, but will |
31 | /// abort as soon as possible. |
32 | pub fail_fast: bool, |
33 | pub options: Options, |
34 | } |
35 | |
36 | impl TestOpts { |
37 | pub fn use_color(&self) -> bool { |
38 | match self.color { |
39 | ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(), |
40 | ColorConfig::AlwaysColor => true, |
41 | ColorConfig::NeverColor => false, |
42 | } |
43 | } |
44 | } |
45 | |
46 | /// Result of parsing the options. |
47 | pub type OptRes = Result<TestOpts, String>; |
48 | /// Result of parsing the option part. |
49 | type OptPartRes<T> = Result<T, String>; |
50 | |
51 | fn optgroups() -> getopts::Options { |
52 | let mut opts = getopts::Options::new(); |
53 | opts.optflag("" , "include-ignored" , "Run ignored and not ignored tests" ) |
54 | .optflag("" , "ignored" , "Run only ignored tests" ) |
55 | .optflag("" , "force-run-in-process" , "Forces tests to run in-process when panic=abort" ) |
56 | .optflag("" , "exclude-should-panic" , "Excludes tests marked as should_panic" ) |
57 | .optflag("" , "test" , "Run tests and not benchmarks" ) |
58 | .optflag("" , "bench" , "Run benchmarks instead of tests" ) |
59 | .optflag("" , "list" , "List all tests and benchmarks" ) |
60 | .optflag("h" , "help" , "Display this message" ) |
61 | .optopt("" , "logfile" , "Write logs to the specified file" , "PATH" ) |
62 | .optflag( |
63 | "" , |
64 | "nocapture" , |
65 | "don't capture stdout/stderr of each \ |
66 | task, allow printing directly" , |
67 | ) |
68 | .optopt( |
69 | "" , |
70 | "test-threads" , |
71 | "Number of threads used for running tests \ |
72 | in parallel" , |
73 | "n_threads" , |
74 | ) |
75 | .optmulti( |
76 | "" , |
77 | "skip" , |
78 | "Skip tests whose names contain FILTER (this flag can \ |
79 | be used multiple times)" , |
80 | "FILTER" , |
81 | ) |
82 | .optflag( |
83 | "q" , |
84 | "quiet" , |
85 | "Display one character per test instead of one line. \ |
86 | Alias to --format=terse" , |
87 | ) |
88 | .optflag("" , "exact" , "Exactly match filters rather than by substring" ) |
89 | .optopt( |
90 | "" , |
91 | "color" , |
92 | "Configure coloring of output: |
93 | auto = colorize if stdout is a tty and tests are run on serially (default); |
94 | always = always colorize output; |
95 | never = never colorize output;" , |
96 | "auto|always|never" , |
97 | ) |
98 | .optopt( |
99 | "" , |
100 | "format" , |
101 | "Configure formatting of output: |
102 | pretty = Print verbose output; |
103 | terse = Display one character per test; |
104 | json = Output a json document; |
105 | junit = Output a JUnit document" , |
106 | "pretty|terse|json|junit" , |
107 | ) |
108 | .optflag("" , "show-output" , "Show captured stdout of successful tests" ) |
109 | .optopt( |
110 | "Z" , |
111 | "" , |
112 | "Enable nightly-only flags: |
113 | unstable-options = Allow use of experimental features" , |
114 | "unstable-options" , |
115 | ) |
116 | .optflag( |
117 | "" , |
118 | "report-time" , |
119 | "Show execution time of each test. |
120 | |
121 | Threshold values for colorized output can be configured via |
122 | `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and |
123 | `RUST_TEST_TIME_DOCTEST` environment variables. |
124 | |
125 | Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`. |
126 | Durations must be specified in milliseconds, e.g. `500,2000` means that the warn time |
127 | is 0.5 seconds, and the critical time is 2 seconds. |
128 | |
129 | Not available for --format=terse" , |
130 | ) |
131 | .optflag( |
132 | "" , |
133 | "ensure-time" , |
134 | "Treat excess of the test execution time limit as error. |
135 | |
136 | Threshold values for this option can be configured via |
137 | `RUST_TEST_TIME_UNIT`, `RUST_TEST_TIME_INTEGRATION` and |
138 | `RUST_TEST_TIME_DOCTEST` environment variables. |
139 | |
140 | Expected format of environment variable is `VARIABLE=WARN_TIME,CRITICAL_TIME`. |
141 | |
142 | `CRITICAL_TIME` here means the limit that should not be exceeded by test. |
143 | " , |
144 | ) |
145 | .optflag("" , "shuffle" , "Run tests in random order" ) |
146 | .optopt( |
147 | "" , |
148 | "shuffle-seed" , |
149 | "Run tests in random order; seed the random number generator with SEED" , |
150 | "SEED" , |
151 | ); |
152 | opts |
153 | } |
154 | |
155 | fn usage(binary: &str, options: &getopts::Options) { |
156 | let message = format!("Usage: {binary} [OPTIONS] [FILTERS...]" ); |
157 | println!( |
158 | r#"{usage} |
159 | |
160 | The FILTER string is tested against the name of all tests, and only those |
161 | tests whose names contain the filter are run. Multiple filter strings may |
162 | be passed, which will run all tests matching any of the filters. |
163 | |
164 | By default, all tests are run in parallel. This can be altered with the |
165 | --test-threads flag or the RUST_TEST_THREADS environment variable when running |
166 | tests (set it to 1). |
167 | |
168 | By default, the tests are run in alphabetical order. Use --shuffle or set |
169 | RUST_TEST_SHUFFLE to run the tests in random order. Pass the generated |
170 | "shuffle seed" to --shuffle-seed (or set RUST_TEST_SHUFFLE_SEED) to run the |
171 | tests in the same order again. Note that --shuffle and --shuffle-seed do not |
172 | affect whether the tests are run in parallel. |
173 | |
174 | All tests have their standard output and standard error captured by default. |
175 | This can be overridden with the --nocapture flag or setting RUST_TEST_NOCAPTURE |
176 | environment variable to a value other than "0". Logging is not captured by default. |
177 | |
178 | Test Attributes: |
179 | |
180 | `#[test]` - Indicates a function is a test to be run. This function |
181 | takes no arguments. |
182 | `#[bench]` - Indicates a function is a benchmark to be run. This |
183 | function takes one argument (test::Bencher). |
184 | `#[should_panic]` - This function (also labeled with `#[test]`) will only pass if |
185 | the code causes a panic (an assertion failure or panic!) |
186 | A message may be provided, which the failure string must |
187 | contain: #[should_panic(expected = "foo")]. |
188 | `#[ignore]` - When applied to a function which is already attributed as a |
189 | test, then the test runner will ignore these tests during |
190 | normal test runs. Running with --ignored or --include-ignored will run |
191 | these tests."# , |
192 | usage = options.usage(&message) |
193 | ); |
194 | } |
195 | |
196 | /// Parses command line arguments into test options. |
197 | /// Returns `None` if help was requested (since we only show help message and don't run tests), |
198 | /// returns `Some(Err(..))` if provided arguments are incorrect, |
199 | /// otherwise creates a `TestOpts` object and returns it. |
200 | pub fn parse_opts(args: &[String]) -> Option<OptRes> { |
201 | // Parse matches. |
202 | let opts = optgroups(); |
203 | let binary: &str = args.get(0).map(|c| &**c).unwrap_or("..." ); |
204 | let args = args.get(1..).unwrap_or(args); |
205 | let matches = match opts.parse(args) { |
206 | Ok(m) => m, |
207 | Err(f) => return Some(Err(f.to_string())), |
208 | }; |
209 | |
210 | // Check if help was requested. |
211 | if matches.opt_present("h" ) { |
212 | // Show help and do nothing more. |
213 | usage(binary, &opts); |
214 | return None; |
215 | } |
216 | |
217 | // Actually parse the opts. |
218 | let opts_result = parse_opts_impl(matches); |
219 | |
220 | Some(opts_result) |
221 | } |
222 | |
223 | // Gets the option value and checks if unstable features are enabled. |
224 | macro_rules! unstable_optflag { |
225 | ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{ |
226 | let opt = $matches.opt_present($option_name); |
227 | if !$allow_unstable && opt { |
228 | return Err(format!( |
229 | "The \"{} \" flag is only accepted on the nightly compiler with -Z unstable-options" , |
230 | $option_name |
231 | )); |
232 | } |
233 | |
234 | opt |
235 | }}; |
236 | } |
237 | |
238 | // Gets the option value and checks if unstable features are enabled. |
239 | macro_rules! unstable_optopt { |
240 | ($matches:ident, $allow_unstable:ident, $option_name:literal) => {{ |
241 | let opt = $matches.opt_str($option_name); |
242 | if !$allow_unstable && opt.is_some() { |
243 | return Err(format!( |
244 | "The \"{} \" option is only accepted on the nightly compiler with -Z unstable-options" , |
245 | $option_name |
246 | )); |
247 | } |
248 | |
249 | opt |
250 | }}; |
251 | } |
252 | |
253 | // Implementation of `parse_opts` that doesn't care about help message |
254 | // and returns a `Result`. |
255 | fn parse_opts_impl(matches: getopts::Matches) -> OptRes { |
256 | let allow_unstable = get_allow_unstable(&matches)?; |
257 | |
258 | // Unstable flags |
259 | let force_run_in_process = unstable_optflag!(matches, allow_unstable, "force-run-in-process" ); |
260 | let exclude_should_panic = unstable_optflag!(matches, allow_unstable, "exclude-should-panic" ); |
261 | let time_options = get_time_options(&matches, allow_unstable)?; |
262 | let shuffle = get_shuffle(&matches, allow_unstable)?; |
263 | let shuffle_seed = get_shuffle_seed(&matches, allow_unstable)?; |
264 | |
265 | let include_ignored = matches.opt_present("include-ignored" ); |
266 | let quiet = matches.opt_present("quiet" ); |
267 | let exact = matches.opt_present("exact" ); |
268 | let list = matches.opt_present("list" ); |
269 | let skip = matches.opt_strs("skip" ); |
270 | |
271 | let bench_benchmarks = matches.opt_present("bench" ); |
272 | let run_tests = !bench_benchmarks || matches.opt_present("test" ); |
273 | |
274 | let logfile = get_log_file(&matches)?; |
275 | let run_ignored = get_run_ignored(&matches, include_ignored)?; |
276 | let filters = matches.free.clone(); |
277 | let nocapture = get_nocapture(&matches)?; |
278 | let test_threads = get_test_threads(&matches)?; |
279 | let color = get_color_config(&matches)?; |
280 | let format = get_format(&matches, quiet, allow_unstable)?; |
281 | |
282 | let options = Options::new().display_output(matches.opt_present("show-output" )); |
283 | |
284 | let test_opts = TestOpts { |
285 | list, |
286 | filters, |
287 | filter_exact: exact, |
288 | force_run_in_process, |
289 | exclude_should_panic, |
290 | run_ignored, |
291 | run_tests, |
292 | bench_benchmarks, |
293 | logfile, |
294 | nocapture, |
295 | color, |
296 | format, |
297 | shuffle, |
298 | shuffle_seed, |
299 | test_threads, |
300 | skip, |
301 | time_options, |
302 | options, |
303 | fail_fast: false, |
304 | }; |
305 | |
306 | Ok(test_opts) |
307 | } |
308 | |
309 | // FIXME: Copied from librustc_ast until linkage errors are resolved. Issue #47566 |
310 | fn is_nightly() -> bool { |
311 | // Whether this is a feature-staged build, i.e., on the beta or stable channel |
312 | let disable_unstable_features = |
313 | option_env!("CFG_DISABLE_UNSTABLE_FEATURES" ).map(|s| s != "0" ).unwrap_or(false); |
314 | // Whether we should enable unstable features for bootstrapping |
315 | let bootstrap: bool = env::var("RUSTC_BOOTSTRAP" ).is_ok(); |
316 | |
317 | bootstrap || !disable_unstable_features |
318 | } |
319 | |
320 | // Gets the CLI options associated with `report-time` feature. |
321 | fn get_time_options( |
322 | matches: &getopts::Matches, |
323 | allow_unstable: bool, |
324 | ) -> OptPartRes<Option<TestTimeOptions>> { |
325 | let report_time: bool = unstable_optflag!(matches, allow_unstable, "report-time" ); |
326 | let ensure_test_time: bool = unstable_optflag!(matches, allow_unstable, "ensure-time" ); |
327 | |
328 | // If `ensure-test-time` option is provided, time output is enforced, |
329 | // so user won't be confused if any of tests will silently fail. |
330 | let options = if report_time || ensure_test_time { |
331 | Some(TestTimeOptions::new_from_env(error_on_excess:ensure_test_time)) |
332 | } else { |
333 | None |
334 | }; |
335 | |
336 | Ok(options) |
337 | } |
338 | |
339 | fn get_shuffle(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<bool> { |
340 | let mut shuffle: bool = unstable_optflag!(matches, allow_unstable, "shuffle" ); |
341 | if !shuffle && allow_unstable { |
342 | shuffle = match env::var("RUST_TEST_SHUFFLE" ) { |
343 | Ok(val) => &val != "0" , |
344 | Err(_) => false, |
345 | }; |
346 | } |
347 | |
348 | Ok(shuffle) |
349 | } |
350 | |
351 | fn get_shuffle_seed(matches: &getopts::Matches, allow_unstable: bool) -> OptPartRes<Option<u64>> { |
352 | let mut shuffle_seed = match unstable_optopt!(matches, allow_unstable, "shuffle-seed" ) { |
353 | Some(n_str) => match n_str.parse::<u64>() { |
354 | Ok(n) => Some(n), |
355 | Err(e) => { |
356 | return Err(format!( |
357 | "argument for --shuffle-seed must be a number \ |
358 | (error: {e})" |
359 | )); |
360 | } |
361 | }, |
362 | None => None, |
363 | }; |
364 | |
365 | if shuffle_seed.is_none() && allow_unstable { |
366 | shuffle_seed = match env::var("RUST_TEST_SHUFFLE_SEED" ) { |
367 | Ok(val) => match val.parse::<u64>() { |
368 | Ok(n) => Some(n), |
369 | Err(_) => panic!("RUST_TEST_SHUFFLE_SEED is `{val}`, should be a number." ), |
370 | }, |
371 | Err(_) => None, |
372 | }; |
373 | } |
374 | |
375 | Ok(shuffle_seed) |
376 | } |
377 | |
378 | fn get_test_threads(matches: &getopts::Matches) -> OptPartRes<Option<usize>> { |
379 | let test_threads: ! = match matches.opt_str("test-threads" ) { |
380 | Some(n_str) => match n_str.parse::<usize>() { |
381 | Ok(0) => return Err("argument for --test-threads must not be 0" .to_string()), |
382 | Ok(n) => Some(n), |
383 | Err(e) => { |
384 | return Err(format!( |
385 | "argument for --test-threads must be a number > 0 \ |
386 | (error: {e})" |
387 | )); |
388 | } |
389 | }, |
390 | None: ! => None, |
391 | }; |
392 | |
393 | Ok(test_threads) |
394 | } |
395 | |
396 | fn get_format( |
397 | matches: &getopts::Matches, |
398 | quiet: bool, |
399 | allow_unstable: bool, |
400 | ) -> OptPartRes<OutputFormat> { |
401 | let format = match matches.opt_str("format" ).as_deref() { |
402 | None if quiet => OutputFormat::Terse, |
403 | Some("pretty" ) | None => OutputFormat::Pretty, |
404 | Some("terse" ) => OutputFormat::Terse, |
405 | Some("json" ) => { |
406 | if !allow_unstable { |
407 | return Err("The \"json \" format is only accepted on the nightly compiler with -Z unstable-options" .into()); |
408 | } |
409 | OutputFormat::Json |
410 | } |
411 | Some("junit" ) => { |
412 | if !allow_unstable { |
413 | return Err("The \"junit \" format is only accepted on the nightly compiler with -Z unstable-options" .into()); |
414 | } |
415 | OutputFormat::Junit |
416 | } |
417 | Some(v) => { |
418 | return Err(format!( |
419 | "argument for --format must be pretty, terse, json or junit (was \ |
420 | {v})" |
421 | )); |
422 | } |
423 | }; |
424 | |
425 | Ok(format) |
426 | } |
427 | |
428 | fn get_color_config(matches: &getopts::Matches) -> OptPartRes<ColorConfig> { |
429 | let color: ColorConfig = match matches.opt_str("color" ).as_deref() { |
430 | Some("auto" ) | None => ColorConfig::AutoColor, |
431 | Some("always" ) => ColorConfig::AlwaysColor, |
432 | Some("never" ) => ColorConfig::NeverColor, |
433 | |
434 | Some(v) => { |
435 | return Err(format!( |
436 | "argument for --color must be auto, always, or never (was \ |
437 | {v})" |
438 | )); |
439 | } |
440 | }; |
441 | |
442 | Ok(color) |
443 | } |
444 | |
445 | fn get_nocapture(matches: &getopts::Matches) -> OptPartRes<bool> { |
446 | let mut nocapture: bool = matches.opt_present("nocapture" ); |
447 | if !nocapture { |
448 | nocapture = match env::var("RUST_TEST_NOCAPTURE" ) { |
449 | Ok(val) => &val != "0" , |
450 | Err(_) => false, |
451 | }; |
452 | } |
453 | |
454 | Ok(nocapture) |
455 | } |
456 | |
457 | fn get_run_ignored(matches: &getopts::Matches, include_ignored: bool) -> OptPartRes<RunIgnored> { |
458 | let run_ignored: RunIgnored = match (include_ignored, matches.opt_present("ignored" )) { |
459 | (true, true) => { |
460 | return Err("the options --include-ignored and --ignored are mutually exclusive" .into()); |
461 | } |
462 | (true, false) => RunIgnored::Yes, |
463 | (false, true) => RunIgnored::Only, |
464 | (false, false) => RunIgnored::No, |
465 | }; |
466 | |
467 | Ok(run_ignored) |
468 | } |
469 | |
470 | fn get_allow_unstable(matches: &getopts::Matches) -> OptPartRes<bool> { |
471 | let mut allow_unstable: bool = false; |
472 | |
473 | if let Some(opt) = matches.opt_str("Z" ) { |
474 | if !is_nightly() { |
475 | return Err("the option `Z` is only accepted on the nightly compiler" .into()); |
476 | } |
477 | |
478 | match &*opt { |
479 | "unstable-options" => { |
480 | allow_unstable = true; |
481 | } |
482 | _ => { |
483 | return Err("Unrecognized option to `Z`" .into()); |
484 | } |
485 | } |
486 | }; |
487 | |
488 | Ok(allow_unstable) |
489 | } |
490 | |
491 | fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> { |
492 | let logfile = matches.opt_str("logfile" ).map(|s| PathBuf::from(&s)); |
493 | |
494 | Ok(logfile) |
495 | } |
496 | |