1 | //! [![github]](https://github.com/dtolnay/trybuild) [![crates-io]](https://crates.io/crates/trybuild) [![docs-rs]](https://docs.rs/trybuild) |
2 | //! |
3 | //! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github |
4 | //! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust |
5 | //! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs |
6 | //! |
7 | //! <br> |
8 | //! |
9 | //! #### A compiler diagnostics testing library in just 3 functions. |
10 | //! |
11 | //! Trybuild is a test harness for invoking rustc on a set of test cases and |
12 | //! asserting that any resulting error messages are the ones intended. |
13 | //! |
14 | //! Such tests are commonly useful for testing error reporting involving |
15 | //! procedural macros. We would write test cases triggering either errors |
16 | //! detected by the macro or errors detected by the Rust compiler in the |
17 | //! resulting expanded code, and compare against the expected errors to ensure |
18 | //! that they remain user-friendly. |
19 | //! |
20 | //! This style of testing is sometimes called *ui tests* because they test |
21 | //! aspects of the user's interaction with a library outside of what would be |
22 | //! covered by ordinary API tests. |
23 | //! |
24 | //! Nothing here is specific to macros; trybuild would work equally well for |
25 | //! testing misuse of non-macro APIs. |
26 | //! |
27 | //! <br> |
28 | //! |
29 | //! # Compile-fail tests |
30 | //! |
31 | //! A minimal trybuild setup looks like this: |
32 | //! |
33 | //! ``` |
34 | //! #[test] |
35 | //! fn ui() { |
36 | //! let t = trybuild::TestCases::new(); |
37 | //! t.compile_fail("tests/ui/*.rs" ); |
38 | //! } |
39 | //! ``` |
40 | //! |
41 | //! The test can be run with `cargo test`. It will individually compile each of |
42 | //! the source files matching the glob pattern, expect them to fail to compile, |
43 | //! and assert that the compiler's error message matches an adjacently named |
44 | //! _*.stderr_ file containing the expected output (same file name as the test |
45 | //! except with a different extension). If it matches, the test case is |
46 | //! considered to succeed. |
47 | //! |
48 | //! Dependencies listed under `[dev-dependencies]` in the project's Cargo.toml |
49 | //! are accessible from within the test cases. |
50 | //! |
51 | //! <p align="center"> |
52 | //! <img src="https://user-images.githubusercontent.com/1940490/57186574-76469e00-6e96-11e9-8cb5-b63b657170c9.png" width="700"> |
53 | //! </p> |
54 | //! |
55 | //! Failing tests display the expected vs actual compiler output inline. |
56 | //! |
57 | //! <p align="center"> |
58 | //! <img src="https://user-images.githubusercontent.com/1940490/57186575-79418e80-6e96-11e9-9478-c9b3dc10327f.png" width="700"> |
59 | //! </p> |
60 | //! |
61 | //! A compile_fail test that fails to fail to compile is also a failure. |
62 | //! |
63 | //! <p align="center"> |
64 | //! <img src="https://user-images.githubusercontent.com/1940490/57186576-7b0b5200-6e96-11e9-8bfd-2de705125108.png" width="700"> |
65 | //! </p> |
66 | //! |
67 | //! <br> |
68 | //! |
69 | //! # Pass tests |
70 | //! |
71 | //! The same test harness is able to run tests that are expected to pass, too. |
72 | //! Ordinarily you would just have Cargo run such tests directly, but being able |
73 | //! to combine modes like this could be useful for workshops in which |
74 | //! participants work through test cases enabling one at a time. Trybuild was |
75 | //! originally developed for my [procedural macros workshop at Rust |
76 | //! Latam][workshop]. |
77 | //! |
78 | //! [workshop]: https://github.com/dtolnay/proc-macro-workshop |
79 | //! |
80 | //! ``` |
81 | //! #[test] |
82 | //! fn ui() { |
83 | //! let t = trybuild::TestCases::new(); |
84 | //! t.pass("tests/01-parse-header.rs" ); |
85 | //! t.pass("tests/02-parse-body.rs" ); |
86 | //! t.compile_fail("tests/03-expand-four-errors.rs" ); |
87 | //! t.pass("tests/04-paste-ident.rs" ); |
88 | //! t.pass("tests/05-repeat-section.rs" ); |
89 | //! //t.pass("tests/06-make-work-in-function.rs"); |
90 | //! //t.pass("tests/07-init-array.rs"); |
91 | //! //t.compile_fail("tests/08-ident-span.rs"); |
92 | //! } |
93 | //! ``` |
94 | //! |
95 | //! Pass tests are considered to succeed if they compile successfully and have a |
96 | //! `main` function that does not panic when the compiled binary is executed. |
97 | //! |
98 | //! <p align="center"> |
99 | //! <img src="https://user-images.githubusercontent.com/1940490/57186580-7f376f80-6e96-11e9-9cae-8257609269ef.png" width="700"> |
100 | //! </p> |
101 | //! |
102 | //! <br> |
103 | //! |
104 | //! # Details |
105 | //! |
106 | //! That's the entire API. |
107 | //! |
108 | //! <br> |
109 | //! |
110 | //! # Workflow |
111 | //! |
112 | //! There are two ways to update the _*.stderr_ files as you iterate on your |
113 | //! test cases or your library; handwriting them is not recommended. |
114 | //! |
115 | //! First, if a test case is being run as compile_fail but a corresponding |
116 | //! _*.stderr_ file does not exist, the test runner will save the actual |
117 | //! compiler output with the right filename into a directory called *wip* within |
118 | //! the directory containing Cargo.toml. So you can update these files by |
119 | //! deleting them, running `cargo test`, and moving all the files from *wip* |
120 | //! into your testcase directory. |
121 | //! |
122 | //! <p align="center"> |
123 | //! <img src="https://user-images.githubusercontent.com/1940490/57186579-7cd51580-6e96-11e9-9f19-54dcecc9fbba.png" width="700"> |
124 | //! </p> |
125 | //! |
126 | //! Alternatively, run `cargo test` with the environment variable |
127 | //! `TRYBUILD=overwrite` to skip the *wip* directory and write all compiler |
128 | //! output directly in place. You'll want to check `git diff` afterward to be |
129 | //! sure the compiler's output is what you had in mind. |
130 | //! |
131 | //! <br> |
132 | //! |
133 | //! # What to test |
134 | //! |
135 | //! When it comes to compile-fail tests, write tests for anything for which you |
136 | //! care to find out when there are changes in the user-facing compiler output. |
137 | //! As a negative example, please don't write compile-fail tests simply calling |
138 | //! all of your public APIs with arguments of the wrong type; there would be no |
139 | //! benefit. |
140 | //! |
141 | //! A common use would be for testing specific targeted error messages emitted |
142 | //! by a procedural macro. For example the derive macro from the [`ref-cast`] |
143 | //! crate is required to be placed on a type that has either `#[repr(C)]` or |
144 | //! `#[repr(transparent)]` in order for the expansion to be free of undefined |
145 | //! behavior, which it enforces at compile time: |
146 | //! |
147 | //! [`ref-cast`]: https://github.com/dtolnay/ref-cast |
148 | //! |
149 | //! ```console |
150 | //! error: RefCast trait requires #[repr(C)] or #[repr(transparent)] |
151 | //! --> $DIR/missing-repr.rs:3:10 |
152 | //! | |
153 | //! 3 | #[derive(RefCast)] |
154 | //! | ^^^^^^^ |
155 | //! ``` |
156 | //! |
157 | //! Macros that consume helper attributes will want to check that unrecognized |
158 | //! content within those attributes is properly indicated to the caller. Is the |
159 | //! error message correctly placed under the erroneous tokens, not on a useless |
160 | //! call\_site span? |
161 | //! |
162 | //! ```console |
163 | //! error: unknown serde field attribute `qqq` |
164 | //! --> $DIR/unknown-attribute.rs:5:13 |
165 | //! | |
166 | //! 5 | #[serde(qqq = "...")] |
167 | //! | ^^^ |
168 | //! ``` |
169 | //! |
170 | //! Declarative macros can benefit from compile-fail tests too. The [`json!`] |
171 | //! macro from serde\_json is just a great big macro\_rules macro but makes an |
172 | //! effort to have error messages from broken JSON in the input always appear on |
173 | //! the most appropriate token: |
174 | //! |
175 | //! [`json!`]: https://docs.rs/serde_json/1.0/serde_json/macro.json.html |
176 | //! |
177 | //! ```console |
178 | //! error: no rules expected the token `,` |
179 | //! --> $DIR/double-comma.rs:4:38 |
180 | //! | |
181 | //! 4 | println!("{}", json!({ "k": null,, })); |
182 | //! | ^ no rules expected this token in macro call |
183 | //! ``` |
184 | //! |
185 | //! Sometimes we may have a macro that expands successfully but we count on it |
186 | //! to trigger particular compiler errors at some point beyond macro expansion. |
187 | //! For example the [`readonly`] crate introduces struct fields that are public |
188 | //! but readable only, even if the caller has a &mut reference to the |
189 | //! surrounding struct. If someone writes to a readonly field, we need to be |
190 | //! sure that it wouldn't compile: |
191 | //! |
192 | //! [`readonly`]: https://github.com/dtolnay/readonly |
193 | //! |
194 | //! ```console |
195 | //! error[E0594]: cannot assign to data in a `&` reference |
196 | //! --> $DIR/write-a-readonly.rs:17:26 |
197 | //! | |
198 | //! 17 | println!("{}", s.n); s.n += 1; |
199 | //! | ^^^^^^^^ cannot assign |
200 | //! ``` |
201 | //! |
202 | //! In all of these cases, the compiler's output can change because our crate or |
203 | //! one of our dependencies broke something, or as a consequence of changes in |
204 | //! the Rust compiler. Both are good reasons to have well conceived compile-fail |
205 | //! tests. If we refactor and mistakenly cause an error that used to be correct |
206 | //! to now no longer be emitted or be emitted in the wrong place, that is |
207 | //! important for a test suite to catch. If the compiler changes something that |
208 | //! makes error messages that we care about substantially worse, it is also |
209 | //! important to catch and report as a compiler issue. |
210 | |
211 | #![doc (html_root_url = "https://docs.rs/trybuild/1.0.85" )] |
212 | #![allow ( |
213 | clippy::collapsible_if, |
214 | clippy::default_trait_access, |
215 | clippy::derive_partial_eq_without_eq, |
216 | clippy::doc_markdown, |
217 | clippy::enum_glob_use, |
218 | clippy::iter_not_returning_iterator, // https://github.com/rust-lang/rust-clippy/issues/8285 |
219 | clippy::let_underscore_untyped, // https://github.com/rust-lang/rust-clippy/issues/10410 |
220 | clippy::manual_assert, |
221 | clippy::manual_range_contains, |
222 | clippy::module_inception, |
223 | clippy::module_name_repetitions, |
224 | clippy::must_use_candidate, |
225 | clippy::needless_pass_by_value, |
226 | clippy::non_ascii_literal, |
227 | clippy::range_plus_one, |
228 | clippy::similar_names, |
229 | clippy::single_match_else, |
230 | clippy::too_many_lines, |
231 | clippy::trivially_copy_pass_by_ref, |
232 | clippy::unused_self, |
233 | clippy::while_let_on_iterator, |
234 | )] |
235 | #![deny (clippy::clone_on_ref_ptr)] |
236 | |
237 | #[macro_use ] |
238 | mod term; |
239 | |
240 | #[macro_use ] |
241 | mod path; |
242 | |
243 | mod cargo; |
244 | mod dependencies; |
245 | mod diff; |
246 | mod directory; |
247 | mod env; |
248 | mod error; |
249 | mod expand; |
250 | mod features; |
251 | mod flock; |
252 | mod inherit; |
253 | mod manifest; |
254 | mod message; |
255 | mod normalize; |
256 | mod run; |
257 | mod rustflags; |
258 | |
259 | use std::cell::RefCell; |
260 | use std::panic::RefUnwindSafe; |
261 | use std::path::{Path, PathBuf}; |
262 | use std::thread; |
263 | |
264 | #[derive(Debug)] |
265 | pub struct TestCases { |
266 | runner: RefCell<Runner>, |
267 | } |
268 | |
269 | #[derive(Debug)] |
270 | struct Runner { |
271 | tests: Vec<Test>, |
272 | } |
273 | |
274 | #[derive(Clone, Debug)] |
275 | struct Test { |
276 | path: PathBuf, |
277 | expected: Expected, |
278 | } |
279 | |
280 | #[derive(Copy, Clone, Debug)] |
281 | enum Expected { |
282 | Pass, |
283 | CompileFail, |
284 | } |
285 | |
286 | impl TestCases { |
287 | #[allow (clippy::new_without_default)] |
288 | pub fn new() -> Self { |
289 | TestCases { |
290 | runner: RefCell::new(Runner { tests: Vec::new() }), |
291 | } |
292 | } |
293 | |
294 | pub fn pass<P: AsRef<Path>>(&self, path: P) { |
295 | self.runner.borrow_mut().tests.push(Test { |
296 | path: path.as_ref().to_owned(), |
297 | expected: Expected::Pass, |
298 | }); |
299 | } |
300 | |
301 | pub fn compile_fail<P: AsRef<Path>>(&self, path: P) { |
302 | self.runner.borrow_mut().tests.push(Test { |
303 | path: path.as_ref().to_owned(), |
304 | expected: Expected::CompileFail, |
305 | }); |
306 | } |
307 | } |
308 | |
309 | impl RefUnwindSafe for TestCases {} |
310 | |
311 | #[doc (hidden)] |
312 | impl Drop for TestCases { |
313 | fn drop(&mut self) { |
314 | if !thread::panicking() { |
315 | self.runner.borrow_mut().run(); |
316 | } |
317 | } |
318 | } |
319 | |