1//! Offers an easy way to build a rustc sysroot from source.
2#![warn(missing_docs)]
3// We prefer to always borrow rather than having to figure out whether we can move or borrow (which
4// depends on whether the variable is used again later).
5#![allow(clippy::needless_borrows_for_generic_args)]
6
7use std::collections::hash_map::DefaultHasher;
8use std::env;
9use std::ffi::{OsStr, OsString};
10use std::fs;
11use std::hash::{Hash, Hasher};
12use std::path::{Path, PathBuf};
13use std::process::Command;
14
15use anyhow::{bail, Context, Result};
16use tempfile::TempDir;
17use walkdir::WalkDir;
18
19/// The name of the profile used for buliding the sysroot.
20const DEFAULT_SYSROOT_PROFILE: &str = "custom_sysroot";
21
22fn rustc_sysroot_dir(mut rustc: Command) -> Result<PathBuf> {
23 let output: Output = rustcResult
24 .args(["--print", "sysroot"])
25 .output()
26 .context("failed to determine sysroot")?;
27 if !output.status.success() {
28 bail!(
29 "failed to determine sysroot; rustc said:\n{}",
30 String::from_utf8_lossy(&output.stderr).trim_end()
31 );
32 }
33 let sysroot: &str =
34 std::str::from_utf8(&output.stdout).context("sysroot folder is not valid UTF-8")?;
35 let sysroot: PathBuf = PathBuf::from(sysroot.trim_end_matches('\n'));
36 if !sysroot.is_dir() {
37 bail!(
38 "sysroot directory `{}` is not a directory",
39 sysroot.display()
40 );
41 }
42 Ok(sysroot)
43}
44
45/// Returns where the given rustc stores its sysroot source code.
46pub fn rustc_sysroot_src(rustc: Command) -> Result<PathBuf> {
47 let sysroot: PathBuf = rustc_sysroot_dir(rustc)?;
48 let rustc_src: PathBuf = sysroot
49 .join("lib")
50 .join("rustlib")
51 .join("src")
52 .join("rust")
53 .join(path:"library");
54 // There could be symlinks here, so better canonicalize to avoid busting the cache due to path
55 // changes.
56 let rustc_src: PathBuf = rustc_src.canonicalize().unwrap_or(default:rustc_src);
57 Ok(rustc_src)
58}
59
60/// Encode a list of rustflags for use in CARGO_ENCODED_RUSTFLAGS.
61pub fn encode_rustflags(flags: &[OsString]) -> OsString {
62 let mut res: OsString = OsString::new();
63 for flag: &OsString in flags {
64 if !res.is_empty() {
65 res.push(OsStr::new("\x1f"));
66 }
67 // Cargo ignores this env var if it's not UTF-8.
68 let flag: &str = flag.to_str().expect(msg:"rustflags must be valid UTF-8");
69 if flag.contains('\x1f') {
70 panic!("rustflags must not contain `\\x1f` separator");
71 }
72 res.push(flag);
73 }
74 res
75}
76
77/// Make a file writeable.
78#[cfg(unix)]
79fn make_writeable(p: &Path) -> Result<()> {
80 // On Unix we avoid `set_readonly(false)`, see
81 // <https://rust-lang.github.io/rust-clippy/master/index.html#permissions_set_readonly_false>.
82 use std::fs::Permissions;
83 use std::os::unix::fs::PermissionsExt;
84
85 let perms: Permissions = fs::metadata(path:p)?.permissions();
86 let perms: Permissions = Permissions::from_mode(perms.mode() | 0o600); // read/write for owner
87 fs::set_permissions(path:p, perm:perms).context("cannot set permissions")?;
88 Ok(())
89}
90
91/// Make a file writeable.
92#[cfg(not(unix))]
93fn make_writeable(p: &Path) -> Result<()> {
94 let mut perms = fs::metadata(p)?.permissions();
95 perms.set_readonly(false);
96 fs::set_permissions(p, perms).context("cannot set permissions")?;
97 Ok(())
98}
99
100/// Hash the metadata and size of every file in a directory, recursively.
101fn hash_recursive(path: &Path, hasher: &mut DefaultHasher) -> Result<()> {
102 // We sort the entries to ensure a stable hash.
103 for entry: Result in WalkDirWalkDir::new(path)
104 .follow_links(yes:true)
105 .sort_by_file_name()
106 .into_iter()
107 {
108 let entry: DirEntry = entry?;
109 // WalkDir yields the directories as well, and File::open will succeed on them. The
110 // reliable way to distinguish directories here is to check explicitly.
111 if entry.file_type().is_dir() {
112 continue;
113 }
114 let meta: Metadata = entry.metadata()?;
115 // Hashing the mtime and file size should catch basically all mutations,
116 // and is faster than hashing the file contents.
117 meta.modified()?.hash(state:hasher);
118 meta.len().hash(state:hasher);
119 }
120 Ok(())
121}
122
123/// The build mode to use for this sysroot.
124#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
125pub enum BuildMode {
126 /// Do a full sysroot build. Suited for all purposes (like the regular sysroot), but only works
127 /// for the host or for targets that have suitable development tools installed.
128 Build,
129 /// Do a check-only sysroot build. This is only suited for check-only builds of crates, but on
130 /// the plus side it works for *arbitrary* targets without having any special tools installed.
131 Check,
132}
133
134impl BuildMode {
135 /// Returns a string with the cargo command matching this build mode.
136 pub fn as_str(&self) -> &str {
137 use BuildMode::*;
138 match self {
139 Build => "build",
140 Check => "check",
141 }
142 }
143}
144
145/// Settings controlling how the sysroot will be built.
146#[derive(Clone, Debug, PartialEq, Eq, Hash)]
147pub enum SysrootConfig {
148 /// Build a no-std (only core and alloc) sysroot.
149 NoStd,
150 /// Build a full sysroot with the `std` and `test` crates.
151 WithStd {
152 /// Features to enable for the `std` crate.
153 std_features: Vec<String>,
154 },
155}
156
157/// Information about a to-be-created sysroot.
158pub struct SysrootBuilder<'a> {
159 sysroot_dir: PathBuf,
160 target: OsString,
161 config: SysrootConfig,
162 mode: BuildMode,
163 rustflags: Vec<OsString>,
164 cargo: Option<Command>,
165 rustc_version: Option<rustc_version::VersionMeta>,
166 when_build_required: Option<Box<dyn FnOnce() + 'a>>,
167}
168
169/// Whether a successful [`SysrootBuilder::build_from_source`] call found a cached sysroot or
170/// built a fresh one.
171#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
172pub enum SysrootStatus {
173 /// The required sysroot is already cached.
174 AlreadyCached,
175 /// A fresh sysroot was just compiled.
176 SysrootBuilt,
177}
178
179/// Hash file name (in target/lib directory).
180const HASH_FILE_NAME: &str = ".rustc-build-sysroot-hash";
181
182impl<'a> SysrootBuilder<'a> {
183 /// Prepare to create a new sysroot in the given folder (that folder should later be passed to
184 /// rustc via `--sysroot`), for the given target.
185 pub fn new(sysroot_dir: &Path, target: impl Into<OsString>) -> Self {
186 let default_flags = &[
187 // This is usually set by bootstrap via `RUSTC_FORCE_UNSTABLE`.
188 "-Zforce-unstable-if-unmarked",
189 // We allow `unexpected_cfgs` as the sysroot has tons of custom `cfg` that rustc does not know about.
190 "-Aunexpected_cfgs",
191 ];
192 SysrootBuilder {
193 sysroot_dir: sysroot_dir.to_owned(),
194 target: target.into(),
195 config: SysrootConfig::WithStd {
196 std_features: vec![],
197 },
198 mode: BuildMode::Build,
199 rustflags: default_flags.iter().map(Into::into).collect(),
200 cargo: None,
201 rustc_version: None,
202 when_build_required: None,
203 }
204 }
205
206 /// Sets the build mode (regular build vs check-only build).
207 pub fn build_mode(mut self, build_mode: BuildMode) -> Self {
208 self.mode = build_mode;
209 self
210 }
211
212 /// Sets the sysroot configuration (which parts of the sysroot to build and with which features).
213 pub fn sysroot_config(mut self, sysroot_config: SysrootConfig) -> Self {
214 self.config = sysroot_config;
215 self
216 }
217
218 /// Appends the given flag.
219 ///
220 /// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`.
221 /// This emulates the usual behavior of Cargo: Lints are normally capped when building
222 /// dependencies, except that they are not capped when building path dependencies, except that
223 /// path dependencies are still capped if they are part of `-Zbuild-std`.
224 pub fn rustflag(mut self, rustflag: impl Into<OsString>) -> Self {
225 self.rustflags.push(rustflag.into());
226 self
227 }
228
229 /// Appends the given flags.
230 ///
231 /// If no `--cap-lints` argument is configured, we will add `--cap-lints=warn`. See
232 /// [`SysrootBuilder::rustflag`] for more explanation.
233 pub fn rustflags(mut self, rustflags: impl IntoIterator<Item = impl Into<OsString>>) -> Self {
234 self.rustflags.extend(rustflags.into_iter().map(Into::into));
235 self
236 }
237
238 /// Sets the cargo command to call.
239 ///
240 /// This will be invoked with `output()`, so if stdout/stderr should be inherited
241 /// then that needs to be set explicitly.
242 pub fn cargo(mut self, cargo: Command) -> Self {
243 self.cargo = Some(cargo);
244 self
245 }
246
247 /// Sets the rustc version information (in case the user has that available).
248 pub fn rustc_version(mut self, rustc_version: rustc_version::VersionMeta) -> Self {
249 self.rustc_version = Some(rustc_version);
250 self
251 }
252
253 /// Sets the hook that will be called if we don't have a cached sysroot available and a new one
254 /// will be compiled.
255 pub fn when_build_required(mut self, when_build_required: impl FnOnce() + 'a) -> Self {
256 self.when_build_required = Some(Box::new(when_build_required));
257 self
258 }
259
260 /// Our configured target can be either a built-in target name, or a path to a target file.
261 /// We use the same logic as rustc to tell which is which:
262 /// https://github.com/rust-lang/rust/blob/8d39ec1825024f3014e1f847942ac5bbfcf055b0/compiler/rustc_session/src/config.rs#L2252-L2263
263 fn target_name(&self) -> &OsStr {
264 let path = Path::new(&self.target);
265 if path.extension().and_then(OsStr::to_str) == Some("json") {
266 // Path::file_stem and Path::extension are the last component of the path split on the
267 // rightmost '.' so if we have an extension we must have a file_stem.
268 path.file_stem().unwrap()
269 } else {
270 // The configured target doesn't end in ".json", so we assume that this is a builtin
271 // target.
272 &self.target
273 }
274 }
275
276 fn sysroot_target_dir(&self) -> PathBuf {
277 self.sysroot_dir
278 .join("lib")
279 .join("rustlib")
280 .join(self.target_name())
281 }
282
283 /// Computes the hash for the sysroot, so that we know whether we have to rebuild.
284 fn sysroot_compute_hash(
285 &self,
286 src_dir: &Path,
287 rustc_version: &rustc_version::VersionMeta,
288 ) -> Result<u64> {
289 let mut hasher = DefaultHasher::new();
290
291 src_dir.hash(&mut hasher);
292 hash_recursive(src_dir, &mut hasher)?;
293 self.config.hash(&mut hasher);
294 self.mode.hash(&mut hasher);
295 self.rustflags.hash(&mut hasher);
296 rustc_version.hash(&mut hasher);
297
298 Ok(hasher.finish())
299 }
300
301 fn sysroot_read_hash(&self) -> Option<u64> {
302 let hash_file = self.sysroot_target_dir().join(HASH_FILE_NAME);
303 let hash = fs::read_to_string(&hash_file).ok()?;
304 hash.parse().ok()
305 }
306
307 /// Generate the contents of the manifest file for the sysroot build.
308 fn gen_manifest(&self, src_dir: &Path) -> String {
309 let have_sysroot_crate = src_dir.join("sysroot").exists();
310 let crates = match &self.config {
311 SysrootConfig::NoStd => format!(
312 r#"
313[dependencies.core]
314path = {src_dir_core:?}
315[dependencies.alloc]
316path = {src_dir_alloc:?}
317[dependencies.compiler_builtins]
318features = ["rustc-dep-of-std", "mem"]
319version = "*"
320 "#,
321 src_dir_core = src_dir.join("core"),
322 src_dir_alloc = src_dir.join("alloc"),
323 ),
324 SysrootConfig::WithStd { std_features } if have_sysroot_crate => format!(
325 r#"
326[dependencies.std]
327features = {std_features:?}
328path = {src_dir_std:?}
329[dependencies.sysroot]
330path = {src_dir_sysroot:?}
331 "#,
332 std_features = std_features,
333 src_dir_std = src_dir.join("std"),
334 src_dir_sysroot = src_dir.join("sysroot"),
335 ),
336 // Fallback for old rustc where the main crate was `test`, not `sysroot`
337 SysrootConfig::WithStd { std_features } => format!(
338 r#"
339[dependencies.std]
340features = {std_features:?}
341path = {src_dir_std:?}
342[dependencies.test]
343path = {src_dir_test:?}
344 "#,
345 std_features = std_features,
346 src_dir_std = src_dir.join("std"),
347 src_dir_test = src_dir.join("test"),
348 ),
349 };
350
351 // If we include a patch for rustc-std-workspace-std for no_std sysroot builds, we get a
352 // warning from Cargo that the patch is unused. If this patching ever breaks that lint will
353 // probably be very helpful, so it would be best to not disable it.
354 // Currently the only user of rustc-std-workspace-alloc is std_detect, which is only used
355 // by std. So we only need to patch rustc-std-workspace-core in no_std sysroot builds, or
356 // that patch also produces a warning.
357 let patches = match &self.config {
358 SysrootConfig::NoStd => format!(
359 r#"
360[patch.crates-io.rustc-std-workspace-core]
361path = {src_dir_workspace_core:?}
362 "#,
363 src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
364 ),
365 SysrootConfig::WithStd { .. } => format!(
366 r#"
367[patch.crates-io.rustc-std-workspace-core]
368path = {src_dir_workspace_core:?}
369[patch.crates-io.rustc-std-workspace-alloc]
370path = {src_dir_workspace_alloc:?}
371[patch.crates-io.rustc-std-workspace-std]
372path = {src_dir_workspace_std:?}
373 "#,
374 src_dir_workspace_core = src_dir.join("rustc-std-workspace-core"),
375 src_dir_workspace_alloc = src_dir.join("rustc-std-workspace-alloc"),
376 src_dir_workspace_std = src_dir.join("rustc-std-workspace-std"),
377 ),
378 };
379
380 format!(
381 r#"
382[package]
383authors = ["rustc-build-sysroot"]
384name = "custom-local-sysroot"
385version = "0.0.0"
386edition = "2018"
387
388[lib]
389# empty dummy, just so that things are being built
390path = "lib.rs"
391
392[profile.{DEFAULT_SYSROOT_PROFILE}]
393# We inherit from the local release profile, but then overwrite some
394# settings to ensure we still get a working sysroot.
395inherits = "release"
396panic = 'unwind'
397
398{crates}
399
400{patches}
401 "#
402 )
403 }
404
405 /// Build the `self` sysroot from the given sources.
406 ///
407 /// `src_dir` must be the `library` source folder, i.e., the one that contains `std/Cargo.toml`.
408 pub fn build_from_source(mut self, src_dir: &Path) -> Result<SysrootStatus> {
409 // A bit of preparation.
410 if !src_dir.join("std").join("Cargo.toml").exists() {
411 bail!(
412 "{:?} does not seem to be a rust library source folder: `src/Cargo.toml` not found",
413 src_dir
414 );
415 }
416 let sysroot_target_dir = self.sysroot_target_dir();
417 let target_name = self.target_name().to_owned();
418 let cargo = self.cargo.take().unwrap_or_else(|| {
419 Command::new(env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")))
420 });
421 let rustc_version = match self.rustc_version.take() {
422 Some(v) => v,
423 None => rustc_version::version_meta()?,
424 };
425
426 // Check if we even need to do anything.
427 let cur_hash = self.sysroot_compute_hash(src_dir, &rustc_version)?;
428 if self.sysroot_read_hash() == Some(cur_hash) {
429 // Already done!
430 return Ok(SysrootStatus::AlreadyCached);
431 }
432
433 // A build is required, so we run the when-build-required function if one was set.
434 if let Some(when_build_required) = self.when_build_required.take() {
435 when_build_required();
436 }
437
438 // Prepare a workspace for cargo
439 let build_dir = TempDir::new().context("failed to create tempdir")?;
440 // Cargo.lock
441 let lock_file = build_dir.path().join("Cargo.lock");
442 let lock_file_src = {
443 // Since <https://github.com/rust-lang/rust/pull/128534>, the lock file
444 // lives inside the src_dir.
445 let new_lock_file_name = src_dir.join("Cargo.lock");
446 if new_lock_file_name.exists() {
447 new_lock_file_name
448 } else {
449 // Previously, the lock file lived one folder up.
450 src_dir
451 .parent()
452 .expect("src_dir must have a parent")
453 .join("Cargo.lock")
454 }
455 };
456 fs::copy(lock_file_src, &lock_file)
457 .context("failed to copy lockfile from sysroot source")?;
458 make_writeable(&lock_file).context("failed to make lockfile writeable")?;
459 // Cargo.toml
460 let manifest_file = build_dir.path().join("Cargo.toml");
461 let manifest = self.gen_manifest(src_dir);
462 fs::write(&manifest_file, manifest.as_bytes()).context("failed to write manifest file")?;
463 // lib.rs
464 let lib_file = build_dir.path().join("lib.rs");
465 let lib = match self.config {
466 SysrootConfig::NoStd => r#"#![no_std]"#,
467 SysrootConfig::WithStd { .. } => "",
468 };
469 fs::write(&lib_file, lib.as_bytes()).context("failed to write lib file")?;
470
471 // Run cargo.
472 let mut cmd = cargo;
473 cmd.arg(self.mode.as_str());
474 cmd.arg("--profile");
475 cmd.arg(DEFAULT_SYSROOT_PROFILE);
476 cmd.arg("--manifest-path");
477 cmd.arg(&manifest_file);
478 cmd.arg("--target");
479 cmd.arg(&self.target);
480 // Set rustflags.
481 cmd.env("CARGO_ENCODED_RUSTFLAGS", encode_rustflags(&self.rustflags));
482 // Make sure the results end up where we expect them.
483 let build_target_dir = build_dir.path().join("target");
484 cmd.env("CARGO_TARGET_DIR", &build_target_dir);
485 // To avoid metadata conflicts, we need to inject some custom data into the crate hash.
486 // bootstrap does the same at
487 // <https://github.com/rust-lang/rust/blob/c8e12cc8bf0de646234524924f39c85d9f3c7c37/src/bootstrap/builder.rs#L1613>.
488 cmd.env("__CARGO_DEFAULT_LIB_METADATA", "rustc-build-sysroot");
489
490 let output = cmd
491 .output()
492 .context("failed to execute cargo for sysroot build")?;
493 if !output.status.success() {
494 let stderr = String::from_utf8_lossy(&output.stderr);
495 if stderr.is_empty() {
496 bail!("sysroot build failed");
497 } else {
498 bail!("sysroot build failed; stderr:\n{}", stderr);
499 }
500 }
501
502 // Create a staging dir that will become the target sysroot dir (so that we can do the final
503 // installation atomically). By creating this directory inside `sysroot_dir`, we ensure that
504 // it is on the same file system (so `fs::rename`) works. This also means that the mtime of
505 // `sysroot_dir` gets updated, which rustc bootstrap relies on as a signal that a rebuild
506 // happened.
507 fs::create_dir_all(&self.sysroot_dir).context("failed to create sysroot dir")?; // TempDir expects the parent to already exist
508 let staging_dir =
509 TempDir::new_in(&self.sysroot_dir).context("failed to create staging dir")?;
510 // Copy the output to `$staging/lib`.
511 let staging_lib_dir = staging_dir.path().join("lib");
512 fs::create_dir(&staging_lib_dir).context("faiked to create staging/lib dir")?;
513 let out_dir = build_target_dir
514 .join(&target_name)
515 .join(DEFAULT_SYSROOT_PROFILE)
516 .join("deps");
517 for entry in fs::read_dir(&out_dir).context("failed to read cargo out dir")? {
518 let entry = entry.context("failed to read cargo out dir entry")?;
519 assert!(
520 entry.file_type().unwrap().is_file(),
521 "cargo out dir must not contain directories"
522 );
523 let entry = entry.path();
524 fs::copy(&entry, staging_lib_dir.join(entry.file_name().unwrap()))
525 .context("failed to copy cargo out file")?;
526 }
527
528 // Write the hash file (into the staging dir).
529 fs::write(
530 staging_dir.path().join(HASH_FILE_NAME),
531 cur_hash.to_string().as_bytes(),
532 )
533 .context("failed to write hash file")?;
534
535 // Atomic copy to final destination via rename.
536 if sysroot_target_dir.exists() {
537 // Remove potentially outdated files.
538 fs::remove_dir_all(&sysroot_target_dir)
539 .context("failed to clean sysroot target dir")?;
540 }
541 // Create the *parent* directroy so we can move into it.
542 fs::create_dir_all(&sysroot_target_dir.parent().unwrap())
543 .context("failed to create target directory")?;
544 fs::rename(staging_dir.path(), sysroot_target_dir).context("failed installing sysroot")?;
545
546 Ok(SysrootStatus::SysrootBuilt)
547 }
548}
549