1use crate::cargo::{self, Metadata, PackageMetadata};
2use crate::dependencies::{self, Dependency, EditionOrInherit};
3use crate::directory::Directory;
4use crate::env::Update;
5use crate::error::{Error, Result};
6use crate::expand::{expand_globs, ExpandedTest};
7use crate::flock::Lock;
8use crate::manifest::{Bin, Build, Config, Manifest, Name, Package, Workspace};
9use crate::message::{self, Fail, Warn};
10use crate::normalize::{self, Context, Variations};
11use crate::{features, rustflags, Expected, Runner, Test};
12use serde_derive::Deserialize;
13use std::collections::{BTreeMap as Map, BTreeSet as Set};
14use std::env;
15use std::ffi::{OsStr, OsString};
16use std::fs::{self, File};
17use std::mem;
18use std::path::{Path, PathBuf};
19use std::str;
20
21#[derive(Debug)]
22pub struct Project {
23 pub dir: Directory,
24 source_dir: Directory,
25 pub target_dir: Directory,
26 pub name: String,
27 update: Update,
28 pub has_pass: bool,
29 has_compile_fail: bool,
30 pub features: Option<Vec<String>>,
31 pub workspace: Directory,
32 pub path_dependencies: Vec<PathDependency>,
33 manifest: Manifest,
34 pub keep_going: bool,
35}
36
37#[derive(Debug)]
38pub struct PathDependency {
39 pub name: String,
40 pub normalized_path: Directory,
41}
42
43struct Report {
44 failures: usize,
45 created_wip: usize,
46}
47
48impl Runner {
49 pub fn run(&mut self) {
50 let mut tests = expand_globs(&self.tests);
51 filter(&mut tests);
52
53 let (project, _lock) = (|| {
54 let mut project = self.prepare(&tests)?;
55 let lock = Lock::acquire(path!(project.dir / ".lock"))?;
56 self.write(&mut project)?;
57 Ok((project, lock))
58 })()
59 .unwrap_or_else(|err| {
60 message::prepare_fail(err);
61 panic!("tests failed");
62 });
63
64 print!("\n\n");
65
66 let len = tests.len();
67 let mut report = Report {
68 failures: 0,
69 created_wip: 0,
70 };
71
72 if tests.is_empty() {
73 message::no_tests_enabled();
74 } else if project.keep_going && !project.has_pass {
75 report = match self.run_all(&project, tests) {
76 Ok(failures) => failures,
77 Err(err) => {
78 message::test_fail(err);
79 Report {
80 failures: len,
81 created_wip: 0,
82 }
83 }
84 }
85 } else {
86 for test in tests {
87 match test.run(&project) {
88 Ok(Outcome::Passed) => {}
89 Ok(Outcome::CreatedWip) => report.created_wip += 1,
90 Err(err) => {
91 report.failures += 1;
92 message::test_fail(err);
93 }
94 }
95 }
96 }
97
98 print!("\n\n");
99
100 if report.failures > 0 && project.name != "trybuild-tests" {
101 panic!("{} of {} tests failed", report.failures, len);
102 }
103 if report.created_wip > 0 && project.name != "trybuild-tests" {
104 panic!(
105 "successfully created new stderr files for {} test cases",
106 report.created_wip,
107 );
108 }
109 }
110
111 fn prepare(&self, tests: &[ExpandedTest]) -> Result<Project> {
112 let Metadata {
113 target_directory: target_dir,
114 workspace_root: workspace,
115 packages,
116 } = cargo::metadata()?;
117
118 let mut has_pass = false;
119 let mut has_compile_fail = false;
120 for e in tests {
121 match e.test.expected {
122 Expected::Pass => has_pass = true,
123 Expected::CompileFail => has_compile_fail = true,
124 }
125 }
126
127 let source_dir = cargo::manifest_dir()?;
128 let source_manifest = dependencies::get_manifest(&source_dir)?;
129
130 let mut features = features::find();
131
132 let path_dependencies = source_manifest
133 .dependencies
134 .iter()
135 .filter_map(|(name, dep)| {
136 let path = dep.path.as_ref()?;
137 if packages.iter().any(|p| &p.name == name) {
138 // Skip path dependencies coming from the workspace itself
139 None
140 } else {
141 Some(PathDependency {
142 name: name.clone(),
143 normalized_path: path.canonicalize().ok()?,
144 })
145 }
146 })
147 .collect();
148
149 let crate_name = &source_manifest.package.name;
150 let project_dir = path!(target_dir / "tests" / "trybuild" / crate_name /);
151 fs::create_dir_all(&project_dir)?;
152
153 let project_name = format!("{}-tests", crate_name);
154 let manifest = self.make_manifest(
155 &workspace,
156 &project_name,
157 &source_dir,
158 &packages,
159 tests,
160 source_manifest,
161 )?;
162
163 if let Some(enabled_features) = &mut features {
164 enabled_features.retain(|feature| manifest.features.contains_key(feature));
165 }
166
167 Ok(Project {
168 dir: project_dir,
169 source_dir,
170 target_dir,
171 name: project_name,
172 update: Update::env()?,
173 has_pass,
174 has_compile_fail,
175 features,
176 workspace,
177 path_dependencies,
178 manifest,
179 keep_going: false,
180 })
181 }
182
183 fn write(&self, project: &mut Project) -> Result<()> {
184 let manifest_toml = basic_toml::to_string(&project.manifest)?;
185
186 let config = self.make_config();
187 let config_toml = basic_toml::to_string(&config)?;
188
189 fs::create_dir_all(path!(project.dir / ".cargo"))?;
190 fs::write(path!(project.dir / ".cargo" / "config.toml"), config_toml)?;
191 fs::write(path!(project.dir / "Cargo.toml"), manifest_toml)?;
192
193 let main_rs = b"\
194 #![allow(unused_crate_dependencies, missing_docs)]\n\
195 fn main() {}\n\
196 ";
197 fs::write(path!(project.dir / "main.rs"), &main_rs[..])?;
198
199 cargo::build_dependencies(project)?;
200
201 Ok(())
202 }
203
204 fn make_manifest(
205 &self,
206 workspace: &Directory,
207 project_name: &str,
208 source_dir: &Directory,
209 packages: &[PackageMetadata],
210 tests: &[ExpandedTest],
211 source_manifest: dependencies::Manifest,
212 ) -> Result<Manifest> {
213 let crate_name = source_manifest.package.name;
214 let workspace_manifest = dependencies::get_workspace_manifest(workspace);
215
216 let edition = match source_manifest.package.edition {
217 EditionOrInherit::Edition(edition) => edition,
218 EditionOrInherit::Inherit => workspace_manifest
219 .workspace
220 .package
221 .edition
222 .ok_or(Error::NoWorkspaceManifest)?,
223 };
224
225 let mut dependencies = Map::new();
226 dependencies.extend(source_manifest.dependencies);
227 dependencies.extend(source_manifest.dev_dependencies);
228
229 let cargo_toml_path = source_dir.join("Cargo.toml");
230 let mut has_lib_target = true;
231 for package_metadata in packages {
232 if package_metadata.manifest_path == cargo_toml_path {
233 has_lib_target = package_metadata
234 .targets
235 .iter()
236 .any(|target| target.crate_types != ["bin"]);
237 }
238 }
239 if has_lib_target {
240 dependencies.insert(
241 crate_name.clone(),
242 Dependency {
243 version: None,
244 path: Some(source_dir.clone()),
245 optional: false,
246 default_features: false,
247 features: Vec::new(),
248 git: None,
249 branch: None,
250 tag: None,
251 rev: None,
252 workspace: false,
253 rest: Map::new(),
254 },
255 );
256 }
257
258 let mut targets = source_manifest.target;
259 for target in targets.values_mut() {
260 let dev_dependencies = mem::take(&mut target.dev_dependencies);
261 target.dependencies.extend(dev_dependencies);
262 }
263
264 let mut features = source_manifest.features;
265 for (feature, enables) in &mut features {
266 enables.retain(|en| {
267 let dep_name = match en.strip_prefix("dep:") {
268 Some(dep_name) => dep_name,
269 None => return false,
270 };
271 if let Some(Dependency { optional: true, .. }) = dependencies.get(dep_name) {
272 return true;
273 }
274 for target in targets.values() {
275 if let Some(Dependency { optional: true, .. }) =
276 target.dependencies.get(dep_name)
277 {
278 return true;
279 }
280 }
281 false
282 });
283 if has_lib_target {
284 enables.insert(0, format!("{}/{}", crate_name, feature));
285 }
286 }
287
288 let mut manifest = Manifest {
289 package: Package {
290 name: project_name.to_owned(),
291 version: "0.0.0".to_owned(),
292 edition,
293 resolver: source_manifest.package.resolver,
294 publish: false,
295 },
296 features,
297 dependencies,
298 target: targets,
299 bins: Vec::new(),
300 workspace: Some(Workspace {
301 dependencies: workspace_manifest.workspace.dependencies,
302 }),
303 // Within a workspace, only the [patch] and [replace] sections in
304 // the workspace root's Cargo.toml are applied by Cargo.
305 patch: workspace_manifest.patch,
306 replace: workspace_manifest.replace,
307 };
308
309 manifest.bins.push(Bin {
310 name: Name(project_name.to_owned()),
311 path: Path::new("main.rs").to_owned(),
312 });
313
314 for expanded in tests {
315 if expanded.error.is_none() {
316 manifest.bins.push(Bin {
317 name: expanded.name.clone(),
318 path: source_dir.join(&expanded.test.path),
319 });
320 }
321 }
322
323 Ok(manifest)
324 }
325
326 fn make_config(&self) -> Config {
327 Config {
328 build: Build {
329 rustflags: rustflags::make_vec(),
330 },
331 }
332 }
333
334 fn run_all(&self, project: &Project, tests: Vec<ExpandedTest>) -> Result<Report> {
335 let mut report = Report {
336 failures: 0,
337 created_wip: 0,
338 };
339
340 let mut path_map = Map::new();
341 for t in &tests {
342 let src_path = project.source_dir.join(&t.test.path);
343 path_map.insert(src_path, (&t.name, &t.test));
344 }
345
346 let output = cargo::build_all_tests(project)?;
347 let parsed = parse_cargo_json(project, &output.stdout, &path_map);
348 let fallback = Stderr::default();
349
350 for mut t in tests {
351 let show_expected = false;
352 message::begin_test(&t.test, show_expected);
353
354 if t.error.is_none() {
355 t.error = check_exists(&t.test.path).err();
356 }
357
358 if t.error.is_none() {
359 let src_path = project.source_dir.join(&t.test.path);
360 let this_test = parsed.stderrs.get(&src_path).unwrap_or(&fallback);
361 match t.test.check(project, &t.name, this_test, "") {
362 Ok(Outcome::Passed) => {}
363 Ok(Outcome::CreatedWip) => report.created_wip += 1,
364 Err(error) => t.error = Some(error),
365 }
366 }
367
368 if let Some(err) = t.error {
369 report.failures += 1;
370 message::test_fail(err);
371 }
372 }
373
374 Ok(report)
375 }
376}
377
378enum Outcome {
379 Passed,
380 CreatedWip,
381}
382
383impl Test {
384 fn run(&self, project: &Project, name: &Name) -> Result<Outcome> {
385 let show_expected = project.has_pass && project.has_compile_fail;
386 message::begin_test(self, show_expected);
387 check_exists(&self.path)?;
388
389 let mut path_map = Map::new();
390 let src_path = project.source_dir.join(&self.path);
391 path_map.insert(src_path.clone(), (name, self));
392
393 let output = cargo::build_test(project, name)?;
394 let parsed = parse_cargo_json(project, &output.stdout, &path_map);
395 let fallback = Stderr::default();
396 let this_test = parsed.stderrs.get(&src_path).unwrap_or(&fallback);
397 self.check(project, name, this_test, &parsed.stdout)
398 }
399
400 fn check(
401 &self,
402 project: &Project,
403 name: &Name,
404 result: &Stderr,
405 build_stdout: &str,
406 ) -> Result<Outcome> {
407 let check = match self.expected {
408 Expected::Pass => Test::check_pass,
409 Expected::CompileFail => Test::check_compile_fail,
410 };
411
412 check(
413 self,
414 project,
415 name,
416 result.success,
417 build_stdout,
418 &result.stderr,
419 )
420 }
421
422 fn check_pass(
423 &self,
424 project: &Project,
425 name: &Name,
426 success: bool,
427 build_stdout: &str,
428 variations: &Variations,
429 ) -> Result<Outcome> {
430 let preferred = variations.preferred();
431 if !success {
432 message::failed_to_build(preferred);
433 return Err(Error::CargoFail);
434 }
435
436 let mut output = cargo::run_test(project, name)?;
437 output.stdout.splice(..0, build_stdout.bytes());
438 message::output(preferred, &output);
439 if output.status.success() {
440 Ok(Outcome::Passed)
441 } else {
442 Err(Error::RunFailed)
443 }
444 }
445
446 fn check_compile_fail(
447 &self,
448 project: &Project,
449 _name: &Name,
450 success: bool,
451 build_stdout: &str,
452 variations: &Variations,
453 ) -> Result<Outcome> {
454 let preferred = variations.preferred();
455
456 if success {
457 message::should_not_have_compiled();
458 message::fail_output(Fail, build_stdout);
459 message::warnings(preferred);
460 return Err(Error::ShouldNotHaveCompiled);
461 }
462
463 let stderr_path = self.path.with_extension("stderr");
464
465 if !stderr_path.exists() {
466 let outcome = match project.update {
467 Update::Wip => {
468 let wip_dir = Path::new("wip");
469 fs::create_dir_all(wip_dir)?;
470 let gitignore_path = wip_dir.join(".gitignore");
471 fs::write(gitignore_path, "*\n")?;
472 let stderr_name = stderr_path
473 .file_name()
474 .unwrap_or_else(|| OsStr::new("test.stderr"));
475 let wip_path = wip_dir.join(stderr_name);
476 message::write_stderr_wip(&wip_path, &stderr_path, preferred);
477 fs::write(wip_path, preferred).map_err(Error::WriteStderr)?;
478 Outcome::CreatedWip
479 }
480 Update::Overwrite => {
481 message::overwrite_stderr(&stderr_path, preferred);
482 fs::write(stderr_path, preferred).map_err(Error::WriteStderr)?;
483 Outcome::Passed
484 }
485 };
486 message::fail_output(Warn, build_stdout);
487 return Ok(outcome);
488 }
489
490 let expected = fs::read_to_string(&stderr_path)
491 .map_err(Error::ReadStderr)?
492 .replace("\r\n", "\n");
493
494 if variations.any(|stderr| expected == stderr) {
495 message::ok();
496 return Ok(Outcome::Passed);
497 }
498
499 match project.update {
500 Update::Wip => {
501 message::mismatch(&expected, preferred);
502 Err(Error::Mismatch)
503 }
504 Update::Overwrite => {
505 message::overwrite_stderr(&stderr_path, preferred);
506 fs::write(stderr_path, preferred).map_err(Error::WriteStderr)?;
507 Ok(Outcome::Passed)
508 }
509 }
510 }
511}
512
513fn check_exists(path: &Path) -> Result<()> {
514 if path.exists() {
515 return Ok(());
516 }
517 match File::open(path) {
518 Ok(_) => Ok(()),
519 Err(err) => Err(Error::Open(path.to_owned(), err)),
520 }
521}
522
523impl ExpandedTest {
524 fn run(self, project: &Project) -> Result<Outcome> {
525 match self.error {
526 None => self.test.run(project, &self.name),
527 Some(error) => {
528 let show_expected = false;
529 message::begin_test(&self.test, show_expected);
530 Err(error)
531 }
532 }
533 }
534}
535
536// Filter which test cases are run by trybuild.
537//
538// $ cargo test -- ui trybuild=tuple_structs.rs
539//
540// The first argument after `--` must be the trybuild test name i.e. the name of
541// the function that has the #[test] attribute and calls trybuild. That's to get
542// Cargo to run the test at all. The next argument starting with `trybuild=`
543// provides a filename filter. Only test cases whose filename contains the
544// filter string will be run.
545#[allow(clippy::needless_collect)] // false positive https://github.com/rust-lang/rust-clippy/issues/5991
546fn filter(tests: &mut Vec<ExpandedTest>) {
547 let filters = env::args_os()
548 .flat_map(OsString::into_string)
549 .filter_map(|mut arg| {
550 const PREFIX: &str = "trybuild=";
551 if arg.starts_with(PREFIX) && arg != PREFIX {
552 Some(arg.split_off(PREFIX.len()))
553 } else {
554 None
555 }
556 })
557 .collect::<Vec<String>>();
558
559 if filters.is_empty() {
560 return;
561 }
562
563 tests.retain(|t| {
564 filters
565 .iter()
566 .any(|f| t.test.path.to_string_lossy().contains(f))
567 });
568}
569
570#[derive(Deserialize)]
571struct CargoMessage {
572 #[allow(dead_code)]
573 reason: Reason,
574 target: RustcTarget,
575 message: RustcMessage,
576}
577
578#[derive(Deserialize)]
579enum Reason {
580 #[serde(rename = "compiler-message")]
581 CompilerMessage,
582}
583
584#[derive(Deserialize)]
585struct RustcTarget {
586 src_path: PathBuf,
587}
588
589#[derive(Deserialize)]
590struct RustcMessage {
591 rendered: String,
592 level: String,
593}
594
595struct ParsedOutputs {
596 stdout: String,
597 stderrs: Map<PathBuf, Stderr>,
598}
599
600struct Stderr {
601 success: bool,
602 stderr: Variations,
603}
604
605impl Default for Stderr {
606 fn default() -> Self {
607 Stderr {
608 success: true,
609 stderr: Variations::default(),
610 }
611 }
612}
613
614fn parse_cargo_json(
615 project: &Project,
616 stdout: &[u8],
617 path_map: &Map<PathBuf, (&Name, &Test)>,
618) -> ParsedOutputs {
619 let mut map = Map::new();
620 let mut nonmessage_stdout = String::new();
621 let mut remaining = &*String::from_utf8_lossy(stdout);
622 let mut seen = Set::new();
623 while !remaining.is_empty() {
624 let begin = match remaining.find("{\"reason\":") {
625 Some(begin) => begin,
626 None => break,
627 };
628 let (nonmessage, rest) = remaining.split_at(begin);
629 nonmessage_stdout.push_str(nonmessage);
630 let len = match rest.find('\n') {
631 Some(end) => end + 1,
632 None => rest.len(),
633 };
634 let (message, rest) = rest.split_at(len);
635 remaining = rest;
636 if !seen.insert(message) {
637 // Discard duplicate messages. This might no longer be necessary
638 // after https://github.com/rust-lang/rust/issues/106571 is fixed.
639 // Normally rustc would filter duplicates itself and I think this is
640 // a short-lived bug.
641 continue;
642 }
643 if let Ok(de) = serde_json::from_str::<CargoMessage>(message) {
644 if de.message.level != "failure-note" {
645 let (name, test) = match path_map.get(&de.target.src_path) {
646 Some(test) => test,
647 None => continue,
648 };
649 let entry = map
650 .entry(de.target.src_path)
651 .or_insert_with(Stderr::default);
652 if de.message.level == "error" {
653 entry.success = false;
654 }
655 let normalized = normalize::diagnostics(
656 &de.message.rendered,
657 Context {
658 krate: &name.0,
659 source_dir: &project.source_dir,
660 workspace: &project.workspace,
661 input_file: &test.path,
662 target_dir: &project.target_dir,
663 path_dependencies: &project.path_dependencies,
664 },
665 );
666 entry.stderr.concat(&normalized);
667 }
668 }
669 }
670 nonmessage_stdout.push_str(remaining);
671 ParsedOutputs {
672 stdout: nonmessage_stdout,
673 stderrs: map,
674 }
675}
676