1//! Offers an easy way to build a rustc sysroot from source.
2
3// We prefer to always borrow rather than having to figure out whether we can move or borrow (which
4// depends on whether the variable is used again later).
5#![allow(clippy::needless_borrows_for_generic_args)]
6
7use std::collections::hash_map::DefaultHasher;
8use std::env;
9use std::ffi::{OsStr, OsString};
10use std::fs;
11use std::hash::{Hash, Hasher};
12use std::ops::Not;
13use std::path::{Path, PathBuf};
14use std::process::Command;
15
16use anyhow::{bail, Context, Result};
17use tempfile::TempDir;
18
19/// Returns where the given rustc stores its sysroot source code.
20pub fn rustc_sysroot_src(mut rustc: Command) -> Result<PathBuf> {
21 let output = rustc
22 .args(["--print", "sysroot"])
23 .output()
24 .context("failed to determine sysroot")?;
25 if !output.status.success() {
26 bail!(
27 "failed to determine sysroot; rustc said:\n{}",
28 String::from_utf8_lossy(&output.stderr).trim_end()
29 );
30 }
31
32 let sysroot =
33 std::str::from_utf8(&output.stdout).context("sysroot folder is not valid UTF-8")?;
34 let sysroot = Path::new(sysroot.trim_end_matches('\n'));
35 let rustc_src = sysroot
36 .join("lib")
37 .join("rustlib")
38 .join("src")
39 .join("rust")
40 .join("library");
41 // There could be symlinks here, so better canonicalize to avoid busting the cache due to path
42 // changes.
43 let rustc_src = rustc_src.canonicalize().unwrap_or(rustc_src);
44 Ok(rustc_src)
45}
46
47/// Encode a list of rustflags for use in CARGO_ENCODED_RUSTFLAGS.
48pub fn encode_rustflags(flags: &[OsString]) -> OsString {
49 let mut res: OsString = OsString::new();
50 for flag: &OsString in flags {
51 if !res.is_empty() {
52 res.push(OsStr::new("\x1f"));
53 }
54 // Cargo ignores this env var if it's not UTF-8.
55 let flag: &str = flag.to_str().expect(msg:"rustflags must be valid UTF-8");
56 if flag.contains('\x1f') {
57 panic!("rustflags must not contain `\\x1f` separator");
58 }
59 res.push(flag);
60 }
61 res
62}
63
64/// Make a file writeable.
65#[cfg(unix)]
66fn make_writeable(p: &Path) -> Result<()> {
67 // On Unix we avoid `set_readonly(false)`, see
68 // <https://rust-lang.github.io/rust-clippy/master/index.html#permissions_set_readonly_false>.
69 use std::fs::Permissions;
70 use std::os::unix::fs::PermissionsExt;
71
72 let perms: Permissions = fs::metadata(path:p)?.permissions();
73 let perms: Permissions = Permissions::from_mode(perms.mode() | 0o600); // read/write for owner
74 fs::set_permissions(path:p, perm:perms).context("cannot set permissions")?;
75 Ok(())
76}
77
78/// Make a file writeable.
79#[cfg(not(unix))]
80fn make_writeable(p: &Path) -> Result<()> {
81 let mut perms = fs::metadata(p)?.permissions();
82 perms.set_readonly(false);
83 fs::set_permissions(p, perms).context("cannot set permissions")?;
84 Ok(())
85}
86
87/// The build mode to use for this sysroot.
88#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
89pub enum BuildMode {
90 /// Do a full sysroot build. Suited for all purposes (like the regular sysroot), but only works
91 /// for the host or for targets that have suitable development tools installed.
92 Build,
93 /// Do a check-only sysroot build. This is only suited for check-only builds of crates, but on
94 /// the plus side it works for *arbitrary* targets without having any special tools installed.
95 Check,
96}
97
98impl BuildMode {
99 pub fn as_str(&self) -> &str {
100 use BuildMode::*;
101 match self {
102 Build => "build",
103 Check => "check",
104 }
105 }
106}
107
108/// Settings controlling how the sysroot will be built.
109#[derive(Clone, Debug, PartialEq, Eq, Hash)]
110pub enum SysrootConfig {
111 /// Build a no-std (only core and alloc) sysroot.
112 NoStd,
113 /// Build a full sysroot with the `std` and `test` crates.
114 WithStd {
115 /// Features to enable for the `std` crate.
116 std_features: Vec<String>,
117 },
118}
119
120/// Information about a to-be-created sysroot.
121pub struct SysrootBuilder {
122 sysroot_dir: PathBuf,
123 target: OsString,
124 config: SysrootConfig,
125 mode: BuildMode,
126 rustflags: Vec<OsString>,
127 cargo: Option<Command>,
128 rustc_version: Option<rustc_version::VersionMeta>,
129}
130
131/// Hash file name (in target/lib directory).
132const HASH_FILE_NAME: &str = ".rustc-build-sysroot-hash";
133
134impl SysrootBuilder {
135 /// Prepare to create a new sysroot in the given folder (that folder should later be passed to
136 /// rustc via `--sysroot`), for the given target.
137 pub fn new(sysroot_dir: &Path, target: impl Into<OsString>) -> Self {
138 SysrootBuilder {
139 sysroot_dir: sysroot_dir.to_owned(),
140 target: target.into(),
141 config: SysrootConfig::WithStd {
142 std_features: vec![],
143 },
144 mode: BuildMode::Build,
145 rustflags: vec![],
146 cargo: None,
147 rustc_version: None,
148 }
149 }
150
151 /// Sets the build mode (regular build vs check-only build).
152 pub fn build_mode(mut self, build_mode: BuildMode) -> Self {
153 self.mode = build_mode;
154 self
155 }
156
157 /// Sets the sysroot configuration (which parts of the sysroot to build and with which features).
158 pub fn sysroot_config(mut self, sysroot_config: SysrootConfig) -> Self {
159 self.config = sysroot_config;
160 self
161 }
162
163 /// Appends the given flag.
164 ///
165 /// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`.
166 /// This emulates the usual behavior of Cargo: Lints are normally capped when building
167 /// dependencies, except that they are not capped when building path dependencies, except that
168 /// path dependencies are still capped if they are part of `-Zbuild-std`.
169 pub fn rustflag(mut self, rustflag: impl Into<OsString>) -> Self {
170 self.rustflags.push(rustflag.into());
171 self
172 }
173
174 /// Appends the given flags.
175 ///
176 /// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`. See
177 /// [`SysrootBuilder::rustflag`] for more explanation.
178 pub fn rustflags(mut self, rustflags: impl IntoIterator<Item = impl Into<OsString>>) -> Self {
179 self.rustflags.extend(rustflags.into_iter().map(Into::into));
180 self
181 }
182
183 /// Sets the cargo command to call.
184 pub fn cargo(mut self, cargo: Command) -> Self {
185 self.cargo = Some(cargo);
186 self
187 }
188
189 /// Sets the rustc version information (in case the user has that available).
190 pub fn rustc_version(mut self, rustc_version: rustc_version::VersionMeta) -> Self {
191 self.rustc_version = Some(rustc_version);
192 self
193 }
194
195 /// Our configured target can be either a built-in target name, or a path to a target file.
196 /// We use the same logic as rustc to tell which is which:
197 /// https://github.com/rust-lang/rust/blob/8d39ec1825024f3014e1f847942ac5bbfcf055b0/compiler/rustc_session/src/config.rs#L2252-L2263
198 fn target_name(&self) -> &OsStr {
199 let path = Path::new(&self.target);
200 if path.extension().and_then(OsStr::to_str) == Some("json") {
201 // Path::file_stem and Path::extension are the last component of the path split on the
202 // rightmost '.' so if we have an extension we must have a file_stem.
203 path.file_stem().unwrap()
204 } else {
205 // The configured target doesn't end in ".json", so we assume that this is a builtin
206 // target.
207 &self.target
208 }
209 }
210
211 fn target_sysroot_dir(&self) -> PathBuf {
212 self.sysroot_dir
213 .join("lib")
214 .join("rustlib")
215 .join(self.target_name())
216 }
217
218 /// Computes the hash for the sysroot, so that we know whether we have to rebuild.
219 fn sysroot_compute_hash(
220 &self,
221 src_dir: &Path,
222 rustc_version: &rustc_version::VersionMeta,
223 ) -> u64 {
224 let mut hasher = DefaultHasher::new();
225
226 // For now, we just hash in the information we have in `self`.
227 // Ideally we'd recursively hash the entire folder but that sounds slow?
228 src_dir.hash(&mut hasher);
229 self.config.hash(&mut hasher);
230 self.mode.hash(&mut hasher);
231 self.rustflags.hash(&mut hasher);
232 rustc_version.hash(&mut hasher);
233
234 hasher.finish()
235 }
236
237 fn sysroot_read_hash(&self) -> Option<u64> {
238 let hash_file = self.target_sysroot_dir().join("lib").join(HASH_FILE_NAME);
239 let hash = fs::read_to_string(&hash_file).ok()?;
240 hash.parse().ok()
241 }
242
243 /// Generate the contents of the manifest file for the sysroot build.
244 fn gen_manifest(&self, src_dir: &Path) -> String {
245 let have_sysroot_crate = src_dir.join("sysroot").exists();
246 let crates = match &self.config {
247 SysrootConfig::NoStd => format!(
248 r#"
249[dependencies.core]
250path = {src_dir_core:?}
251[dependencies.alloc]
252path = {src_dir_alloc:?}
253[dependencies.compiler_builtins]
254features = ["rustc-dep-of-std", "mem"]
255version = "*"
256 "#,
257 src_dir_core = src_dir.join("core"),
258 src_dir_alloc = src_dir.join("alloc"),
259 ),
260 SysrootConfig::WithStd { std_features } if have_sysroot_crate => format!(
261 r#"
262[dependencies.std]
263features = {std_features:?}
264path = {src_dir_std:?}
265[dependencies.sysroot]
266path = {src_dir_sysroot:?}
267 "#,
268 std_features = std_features,
269 src_dir_std = src_dir.join("std"),
270 src_dir_sysroot = src_dir.join("sysroot"),
271 ),
272 // Fallback for old rustc where the main crate was `test`, not `sysroot`
273 SysrootConfig::WithStd { std_features } => format!(
274 r#"
275[dependencies.std]
276features = {std_features:?}
277path = {src_dir_std:?}
278[dependencies.test]
279path = {src_dir_test:?}
280 "#,
281 std_features = std_features,
282 src_dir_std = src_dir.join("std"),
283 src_dir_test = src_dir.join("test"),
284 ),
285 };
286
287 // If we include a patch for rustc-std-workspace-std for no_std sysroot builds, we get a
288 // warning from Cargo that the patch is unused. If this patching ever breaks that lint will
289 // probably be very helpful, so it would be best to not disable it.
290 // Currently the only user of rustc-std-workspace-alloc is std_detect, which is only used
291 // by std. So we only need to patch rustc-std-workspace-core in no_std sysroot builds, or
292 // that patch also produces a warning.
293 let patches = match &self.config {
294 SysrootConfig::NoStd => format!(
295 r#"
296[patch.crates-io.rustc-std-workspace-core]
297path = {src_dir_workspace_core:?}
298 "#,
299 src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
300 ),
301 SysrootConfig::WithStd { .. } => format!(
302 r#"
303[patch.crates-io.rustc-std-workspace-core]
304path = {src_dir_workspace_core:?}
305[patch.crates-io.rustc-std-workspace-alloc]
306path = {src_dir_workspace_alloc:?}
307[patch.crates-io.rustc-std-workspace-std]
308path = {src_dir_workspace_std:?}
309 "#,
310 src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
311 src_dir_workspace_alloc = src_dir.join("rustc-std-workspace-alloc"),
312 src_dir_workspace_std = src_dir.join("rustc-std-workspace-std"),
313 ),
314 };
315
316 format!(
317 r#"
318[package]
319authors = ["rustc-build-sysroot"]
320name = "custom-local-sysroot"
321version = "0.0.0"
322
323[lib]
324# empty dummy, just so that things are being built
325path = "lib.rs"
326
327{crates}
328
329{patches}
330 "#
331 )
332 }
333
334 /// Build the `self` sysroot from the given sources.
335 ///
336 /// `src_dir` must be the `library` source folder, i.e., the one that contains `std/Cargo.toml`.
337 pub fn build_from_source(mut self, src_dir: &Path) -> Result<()> {
338 // A bit of preparation.
339 if !src_dir.join("std").join("Cargo.toml").exists() {
340 bail!(
341 "{:?} does not seem to be a rust library source folder: `src/Cargo.toml` not found",
342 src_dir
343 );
344 }
345 let sysroot_lib_dir = self.target_sysroot_dir().join("lib");
346 let target_name = self.target_name().to_owned();
347 let cargo = self.cargo.take().unwrap_or_else(|| {
348 Command::new(env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")))
349 });
350 let rustc_version = match self.rustc_version.take() {
351 Some(v) => v,
352 None => rustc_version::version_meta()?,
353 };
354
355 // The whole point of this crate is to build the standard library in nonstandard
356 // configurations, which may trip lints due to untested combinations of cfgs.
357 // Cargo applies --cap-lints=allow or --cap-lints=warn when handling -Zbuild-std, which we
358 // of course are not using:
359 // https://github.com/rust-lang/cargo/blob/2ce45605d9db521b5fd6c1211ce8de6055fdb24e/src/cargo/core/compiler/mod.rs#L899
360 // https://github.com/rust-lang/cargo/blob/2ce45605d9db521b5fd6c1211ce8de6055fdb24e/src/cargo/core/compiler/unit.rs#L102-L109
361 // All the standard library crates are path dependencies, and they also sometimes pull in
362 // separately-maintained crates like backtrace by treating their crate roots as module
363 // roots. If we do not cap lints, we can get lint failures outside core or std.
364 // We cannot set --cap-lints=allow because Cargo needs to parse warnings to understand the
365 // output of --print=file-names for crate-types that the target does not support.
366 if !self.rustflags.iter().any(|flag| {
367 // FIXME: OsStr::as_encoded_bytes is cleaner here
368 flag.to_str()
369 .map_or(false, |f| f.starts_with("--cap-lints"))
370 }) {
371 self.rustflags.push("--cap-lints=warn".into());
372 }
373
374 // Check if we even need to do anything.
375 let cur_hash = self.sysroot_compute_hash(src_dir, &rustc_version);
376 if self.sysroot_read_hash() == Some(cur_hash) {
377 // Already done!
378 return Ok(());
379 }
380
381 // Prepare a workspace for cargo
382 let build_dir = TempDir::new().context("failed to create tempdir")?;
383 let lock_file = build_dir.path().join("Cargo.lock");
384 let manifest_file = build_dir.path().join("Cargo.toml");
385 let lib_file = build_dir.path().join("lib.rs");
386 fs::copy(
387 src_dir
388 .parent()
389 .expect("src_dir must have a parent")
390 .join("Cargo.lock"),
391 &lock_file,
392 )
393 .context("failed to copy lockfile from sysroot source")?;
394 make_writeable(&lock_file).context("failed to make lockfile writeable")?;
395 let manifest = self.gen_manifest(src_dir);
396 fs::write(&manifest_file, manifest.as_bytes()).context("failed to write manifest file")?;
397 let lib = match self.config {
398 SysrootConfig::NoStd => r#"#![no_std]"#,
399 SysrootConfig::WithStd { .. } => "",
400 };
401 fs::write(&lib_file, lib.as_bytes()).context("failed to write lib file")?;
402
403 // Run cargo.
404 let mut cmd = cargo;
405 cmd.arg(self.mode.as_str());
406 cmd.arg("--release");
407 cmd.arg("--manifest-path");
408 cmd.arg(&manifest_file);
409 cmd.arg("--target");
410 cmd.arg(&self.target);
411 // Set rustflags.
412 let mut flags = self.rustflags;
413 flags.push("-Zforce-unstable-if-unmarked".into());
414 cmd.env("CARGO_ENCODED_RUSTFLAGS", encode_rustflags(&flags));
415 // Make sure the results end up where we expect them.
416 let build_target_dir = build_dir.path().join("target");
417 cmd.env("CARGO_TARGET_DIR", &build_target_dir);
418 // To avoid metadata conflicts, we need to inject some custom data into the crate hash.
419 // bootstrap does the same at
420 // <https://github.com/rust-lang/rust/blob/c8e12cc8bf0de646234524924f39c85d9f3c7c37/src/bootstrap/builder.rs#L1613>.
421 cmd.env("__CARGO_DEFAULT_LIB_METADATA", "rustc-build-sysroot");
422
423 if cmd
424 .status()
425 .unwrap_or_else(|_| panic!("failed to execute cargo for sysroot build"))
426 .success()
427 .not()
428 {
429 bail!("sysroot build failed");
430 }
431
432 // Copy the output to a staging dir (so that we can do the final installation atomically.)
433 fs::create_dir_all(&self.sysroot_dir).context("failed to create sysroot dir")?; // TempDir expects the parent to already exist
434 let staging_dir =
435 TempDir::new_in(&self.sysroot_dir).context("failed to create staging dir")?;
436 let out_dir = build_target_dir
437 .join(&target_name)
438 .join("release")
439 .join("deps");
440 for entry in fs::read_dir(&out_dir).context("failed to read cargo out dir")? {
441 let entry = entry.context("failed to read cargo out dir entry")?;
442 assert!(
443 entry.file_type().unwrap().is_file(),
444 "cargo out dir must not contain directories"
445 );
446 let entry = entry.path();
447 fs::copy(&entry, staging_dir.path().join(entry.file_name().unwrap()))
448 .context("failed to copy cargo out file")?;
449 }
450
451 // Write the hash file (into the staging dir).
452 fs::write(
453 staging_dir.path().join(HASH_FILE_NAME),
454 cur_hash.to_string().as_bytes(),
455 )
456 .context("failed to write hash file")?;
457
458 // Atomic copy to final destination via rename.
459 if sysroot_lib_dir.exists() {
460 // Remove potentially outdated files.
461 fs::remove_dir_all(&sysroot_lib_dir).context("failed to clean sysroot target dir")?;
462 }
463 fs::create_dir_all(
464 sysroot_lib_dir
465 .parent()
466 .expect("target/lib dir must have a parent"),
467 )
468 .context("failed to create target directory")?;
469 fs::rename(staging_dir.path(), sysroot_lib_dir).context("failed installing sysroot")?;
470
471 Ok(())
472 }
473}
474