1//! This module allows you to configure the default settings for all tests.
2//!
3//! All data structures here are normally parsed from `@` comments
4//! in the files. These comments still overwrite the defaults, although
5//! some boolean settings have no way to disable them.
6
7use crate::build_manager::BuildManager;
8use crate::custom_flags::Flag;
9pub use crate::diagnostics::Level;
10use crate::diagnostics::{Diagnostics, Message};
11pub use crate::parser::{Comments, Condition, Revisioned};
12use crate::parser::{ErrorMatch, ErrorMatchKind, OptWithLine};
13use crate::status_emitter::{SilentStatus, TestStatus};
14use crate::test_result::{Errored, TestOk, TestResult};
15use crate::{core::strip_path_prefix, Config, Error, Errors};
16use spanned::Spanned;
17use std::collections::btree_map::Entry;
18use std::collections::BTreeMap;
19use std::num::NonZeroUsize;
20use std::path::PathBuf;
21use std::process::{Command, Output};
22use std::sync::Arc;
23
24/// All information needed to run a single test
25pub struct TestConfig {
26 /// The generic config for all tests
27 pub config: Config,
28 pub(crate) comments: Arc<Comments>,
29 /// The path to the folder where to look for aux files
30 pub aux_dir: PathBuf,
31 /// When doing long-running operations, you can inform the user about it here.
32 pub status: Box<dyn TestStatus>,
33}
34
35impl TestConfig {
36 /// Create a config for running one file.
37 pub fn one_off_runner(config: Config, path: PathBuf) -> Self {
38 Self {
39 comments: Arc::new(config.comment_defaults.clone()),
40 config,
41 aux_dir: PathBuf::new(),
42 status: Box::new(SilentStatus {
43 revision: "".into(),
44 path,
45 }),
46 }
47 }
48
49 pub(crate) fn patch_out_dir(&mut self) {
50 // Put aux builds into a separate directory per path so that multiple aux files
51 // from different directories (but with the same file name) don't collide.
52 let relative =
53 strip_path_prefix(self.status.path().parent().unwrap(), &self.config.out_dir);
54
55 self.config.out_dir.extend(relative);
56 }
57
58 /// Create a file extension that includes the current revision if necessary.
59 pub fn extension(&self, extension: &str) -> String {
60 if self.status.revision().is_empty() {
61 extension.to_string()
62 } else {
63 format!("{}.{extension}", self.status.revision())
64 }
65 }
66
67 /// The test's expected exit status after applying all comments
68 pub fn exit_status(&self) -> Result<Option<Spanned<i32>>, Errored> {
69 self.comments.exit_status(self.status.revision())
70 }
71
72 /// Whether compiler messages require annotations
73 pub fn require_annotations(&self) -> Option<Spanned<bool>> {
74 self.comments.require_annotations(self.status.revision())
75 }
76
77 pub(crate) fn find_one<'a, T: 'a>(
78 &'a self,
79 kind: &str,
80 f: impl Fn(&'a Revisioned) -> OptWithLine<T>,
81 ) -> Result<OptWithLine<T>, Errored> {
82 self.comments
83 .find_one_for_revision(self.status.revision(), kind, f)
84 }
85
86 /// All comments that apply to the current test.
87 pub fn comments(&self) -> impl Iterator<Item = &'_ Revisioned> {
88 self.comments.for_revision(self.status.revision())
89 }
90
91 pub(crate) fn collect<'a, T, I: Iterator<Item = T>, R: FromIterator<T>>(
92 &'a self,
93 f: impl Fn(&'a Revisioned) -> I,
94 ) -> R {
95 self.comments().flat_map(f).collect()
96 }
97
98 /// Apply custom flags (aux builds, dependencies, ...)
99 pub fn apply_custom(
100 &self,
101 cmd: &mut Command,
102 build_manager: &BuildManager,
103 ) -> Result<(), Errored> {
104 let mut all = BTreeMap::new();
105 for rev in self.comments.for_revision(self.status.revision()) {
106 for (&k, flags) in &rev.custom {
107 for flag in &flags.content {
108 match all.entry(k) {
109 Entry::Vacant(v) => _ = v.insert(vec![flag]),
110 Entry::Occupied(mut o) => {
111 if o.get().last().unwrap().must_be_unique() {
112 // Overwrite previous value so that revisions overwrite default settings
113 // FIXME: report an error if multiple revisions conflict
114 assert_eq!(o.get().len(), 1);
115 o.get_mut()[0] = flag;
116 } else {
117 o.get_mut().push(flag);
118 }
119 }
120 }
121 }
122 }
123 }
124 for flags in all.values() {
125 for flag in flags {
126 flag.apply(cmd, self, build_manager)?;
127 }
128 }
129 Ok(())
130 }
131
132 /// Produce the command that will be executed to run the test.
133 pub fn build_command(&self, build_manager: &BuildManager) -> Result<Command, Errored> {
134 let mut cmd = self.config.program.build(&self.config.out_dir);
135 cmd.arg(self.status.path());
136 for r in self.comments() {
137 cmd.args(&r.compile_flags);
138 }
139
140 self.apply_custom(&mut cmd, build_manager)?;
141
142 if let Some(target) = &self.config.target {
143 // Adding a `--target` arg to calls to Cargo will cause target folders
144 // to create a target-specific sub-folder. We can avoid that by just
145 // not passing a `--target` arg if its the same as the host.
146 if !self.config.host_matches_target() {
147 cmd.arg("--target").arg(target);
148 }
149 }
150
151 cmd.envs(self.envs());
152
153 Ok(cmd)
154 }
155
156 pub(crate) fn output_path(&self, kind: &str) -> PathBuf {
157 let ext = self.extension(kind);
158 if self.comments().any(|r| r.stderr_per_bitwidth) {
159 return self
160 .status
161 .path()
162 .with_extension(format!("{}bit.{ext}", self.config.get_pointer_width()));
163 }
164 self.status.path().with_extension(ext)
165 }
166
167 pub(crate) fn normalize(&self, text: &[u8], kind: &str) -> Vec<u8> {
168 let mut text = text.to_owned();
169
170 for (from, to) in self.comments().flat_map(|r| match kind {
171 _ if kind.ends_with("fixed") => &[] as &[_],
172 "stderr" => &r.normalize_stderr,
173 "stdout" => &r.normalize_stdout,
174 _ => unreachable!(),
175 }) {
176 text = from.replace_all(&text, to).into_owned();
177 }
178 text
179 }
180
181 pub(crate) fn check_test_output(&self, errors: &mut Errors, stdout: &[u8], stderr: &[u8]) {
182 // Check output files (if any)
183 // Check output files against actual output
184 self.check_output(stderr, errors, "stderr");
185 self.check_output(stdout, errors, "stdout");
186 }
187
188 pub(crate) fn check_output(&self, output: &[u8], errors: &mut Errors, kind: &str) -> PathBuf {
189 let path = self.output_path(kind);
190 (self.config.output_conflict_handling)(&path, output, errors, self);
191 path
192 }
193
194 /// Read diagnostics from a test's output.
195 pub fn process(&self, stderr: &[u8]) -> Diagnostics {
196 (self.config.diagnostic_extractor)(self.status.path(), stderr)
197 }
198
199 fn check_test_result(&self, command: &Command, output: Output) -> Result<Output, Errored> {
200 let mut errors = vec![];
201 errors.extend(self.ok(output.status)?);
202 // Always remove annotation comments from stderr.
203 let diagnostics = self.process(&output.stderr);
204 self.check_test_output(&mut errors, &output.stdout, &diagnostics.rendered);
205 // Check error annotations in the source against output
206 self.check_annotations(
207 diagnostics.messages,
208 diagnostics.messages_from_unknown_file_or_line,
209 &mut errors,
210 )?;
211 if errors.is_empty() {
212 Ok(output)
213 } else {
214 Err(Errored {
215 command: format!("{command:?}"),
216 errors,
217 stderr: diagnostics.rendered,
218 stdout: output.stdout,
219 })
220 }
221 }
222
223 pub(crate) fn check_annotations(
224 &self,
225 mut messages: Vec<Vec<Message>>,
226 mut messages_from_unknown_file_or_line: Vec<Message>,
227 errors: &mut Errors,
228 ) -> Result<(), Errored> {
229 let error_patterns = self.comments().flat_map(|r| r.error_in_other_files.iter());
230
231 let mut seen_error_match = None;
232 for error_pattern in error_patterns {
233 seen_error_match = Some(error_pattern.span());
234 // first check the diagnostics messages outside of our file. We check this first, so that
235 // you can mix in-file annotations with //@error-in-other-file annotations, even if there is overlap
236 // in the messages.
237 if let Some(i) = messages_from_unknown_file_or_line
238 .iter()
239 .position(|msg| error_pattern.matches(&msg.message))
240 {
241 messages_from_unknown_file_or_line.remove(i);
242 } else {
243 errors.push(Error::PatternNotFound {
244 pattern: error_pattern.clone(),
245 expected_line: None,
246 });
247 }
248 }
249 let diagnostic_code_prefix = self
250 .find_one("diagnostic_code_prefix", |r| {
251 r.diagnostic_code_prefix.clone()
252 })?
253 .into_inner()
254 .map(|s| s.content)
255 .unwrap_or_default();
256
257 // The order on `Level` is such that `Error` is the highest level.
258 // We will ensure that *all* diagnostics of level at least `lowest_annotation_level`
259 // are matched.
260 let mut lowest_annotation_level = Level::Error;
261 'err: for &ErrorMatch { ref kind, line } in
262 self.comments().flat_map(|r| r.error_matches.iter())
263 {
264 match kind {
265 ErrorMatchKind::Code(code) => {
266 seen_error_match = Some(code.span());
267 }
268 &ErrorMatchKind::Pattern { ref pattern, level } => {
269 seen_error_match = Some(pattern.span());
270 // If we found a diagnostic with a level annotation, make sure that all
271 // diagnostics of that level have annotations, even if we don't end up finding a matching diagnostic
272 // for this pattern.
273 if lowest_annotation_level > level {
274 lowest_annotation_level = level;
275 }
276 }
277 }
278
279 if let Some(msgs) = messages.get_mut(line.get()) {
280 match kind {
281 &ErrorMatchKind::Pattern { ref pattern, level } => {
282 let found = msgs
283 .iter()
284 .position(|msg| pattern.matches(&msg.message) && msg.level == level);
285 if let Some(found) = found {
286 msgs.remove(found);
287 continue;
288 }
289 }
290 ErrorMatchKind::Code(code) => {
291 for (i, msg) in msgs.iter().enumerate() {
292 if msg.level != Level::Error {
293 continue;
294 }
295 let Some(msg_code) = &msg.code else { continue };
296 let Some(msg) = msg_code.strip_prefix(&diagnostic_code_prefix) else {
297 continue;
298 };
299 if msg == **code {
300 msgs.remove(i);
301 continue 'err;
302 }
303 }
304 }
305 }
306 }
307
308 errors.push(match kind {
309 ErrorMatchKind::Pattern { pattern, .. } => Error::PatternNotFound {
310 pattern: pattern.clone(),
311 expected_line: Some(line),
312 },
313 ErrorMatchKind::Code(code) => Error::CodeNotFound {
314 code: Spanned::new(
315 format!("{}{}", diagnostic_code_prefix, **code),
316 code.span(),
317 ),
318 expected_line: Some(line),
319 },
320 });
321 }
322
323 let required_annotation_level = self
324 .find_one("`require_annotations_for_level` annotations", |r| {
325 r.require_annotations_for_level.clone()
326 })?;
327
328 let required_annotation_level = required_annotation_level
329 .into_inner()
330 .map_or(lowest_annotation_level, |l| *l);
331 let filter = |mut msgs: Vec<Message>| -> Vec<_> {
332 msgs.retain(|msg| msg.level >= required_annotation_level);
333 msgs
334 };
335
336 let require_annotations = self.require_annotations();
337
338 if let Some(Spanned { content: true, .. }) = require_annotations {
339 let messages_from_unknown_file_or_line = filter(messages_from_unknown_file_or_line);
340 if !messages_from_unknown_file_or_line.is_empty() {
341 errors.push(Error::ErrorsWithoutPattern {
342 path: None,
343 msgs: messages_from_unknown_file_or_line,
344 });
345 }
346
347 for (line, msgs) in messages.into_iter().enumerate() {
348 let msgs = filter(msgs);
349 if !msgs.is_empty() {
350 let line = NonZeroUsize::new(line).expect("line 0 is always empty");
351 errors.push(Error::ErrorsWithoutPattern {
352 path: Some((self.status.path().to_path_buf(), line)),
353 msgs,
354 });
355 }
356 }
357 }
358
359 match (require_annotations, seen_error_match) {
360 (
361 Some(Spanned {
362 content: false,
363 span: mode,
364 }),
365 Some(span),
366 ) => errors.push(Error::PatternFoundInPassTest { mode, span }),
367 (Some(Spanned { content: true, .. }), None) => errors.push(Error::NoPatternsFound),
368 _ => {}
369 }
370 Ok(())
371 }
372
373 pub(crate) fn run_test(&mut self, build_manager: &Arc<BuildManager>) -> TestResult {
374 self.patch_out_dir();
375
376 let mut cmd = self.build_command(build_manager)?;
377 let stdin = self.status.path().with_extension(self.extension("stdin"));
378 if stdin.exists() {
379 cmd.stdin(std::fs::File::open(stdin).unwrap());
380 }
381
382 let output = build_manager.config.run_command(&mut cmd)?;
383
384 let output = self.check_test_result(&cmd, output)?;
385
386 for rev in self.comments() {
387 for custom in rev.custom.values() {
388 for flag in &custom.content {
389 flag.post_test_action(self, &output, build_manager)?;
390 }
391 }
392 }
393 Ok(TestOk::Ok)
394 }
395
396 pub(crate) fn find_one_custom(&self, arg: &str) -> Result<OptWithLine<&dyn Flag>, Errored> {
397 self.find_one(arg, |r| {
398 r.custom
399 .get(arg)
400 .map(|s| {
401 assert_eq!(s.len(), 1);
402 Spanned::new(&*s[0], s.span())
403 })
404 .into()
405 })
406 }
407
408 pub(crate) fn aborted(&self) -> Result<(), Errored> {
409 self.config.aborted()
410 }
411
412 /// All the environment variables set for the given revision
413 pub fn envs(&self) -> impl Iterator<Item = (&str, &str)> {
414 self.comments()
415 .flat_map(|r| r.env_vars.iter())
416 .map(|(k, v)| (k.as_ref(), v.as_ref()))
417 }
418}
419