From a119803a867a029f8a836949990147e3a7235b8b Mon Sep 17 00:00:00 2001 From: Giang Dao Date: Mon, 7 Oct 2024 23:14:19 +0800 Subject: [PATCH] refactor: Reuse PkgType from build-manifest --- src/bootstrap/Cargo.lock | 89 +++ src/bootstrap/Cargo.toml | 22 +- src/bootstrap/src/core/build_steps/dist.rs | 374 ++++--------- src/bootstrap/src/core/config/config.rs | 1 - src/etc/installer/msi/rust.wxs | 10 +- src/tools/build-manifest/README.md | 2 +- src/tools/build-manifest/src/builder.rs | 402 ++++++++++++++ src/tools/build-manifest/src/checksum.rs | 4 +- src/tools/build-manifest/src/constants.rs | 204 +++++++ src/tools/build-manifest/src/lib.rs | 10 + src/tools/build-manifest/src/main.rs | 610 +-------------------- src/tools/build-manifest/src/versions.rs | 14 +- 12 files changed, 857 insertions(+), 885 deletions(-) create mode 100644 src/tools/build-manifest/src/builder.rs create mode 100644 src/tools/build-manifest/src/constants.rs create mode 100644 src/tools/build-manifest/src/lib.rs diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index efcac4f0953d0..a7952e7ae3581 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -17,6 +23,12 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +[[package]] +name = "anyhow" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" + [[package]] name = "bitflags" version = "2.6.0" @@ -36,6 +48,7 @@ dependencies = [ name = "bootstrap" version = "0.0.0" dependencies = [ + "build-manifest", "build_helper", "cc", "clap", @@ -74,6 +87,22 @@ dependencies = [ "serde", ] +[[package]] +name = "build-manifest" +version = "0.1.0" +dependencies = [ + "anyhow", + "flate2", + "hex", + "rayon", + "serde", + "serde_json", + "sha2", + "tar", + "toml", + "xz2", +] + [[package]] name = "build_helper" version = "0.1.0" @@ -168,6 +197,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -219,6 +257,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "errno" version = "0.3.9" @@ -252,6 +296,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -281,6 +335,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "home" version = "0.5.9" @@ -368,6 +428,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "ntapi" version = "0.4.1" @@ -430,6 +499,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.6" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index ba505089a000a..20c84e3b6121a 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -7,7 +7,7 @@ default-run = "bootstrap" [features] build-metrics = ["sysinfo"] -bootstrap-self-test = [] # enabled in the bootstrap unit tests +bootstrap-self-test = [] # enabled in the bootstrap unit tests [lib] path = "src/lib.rs" @@ -41,13 +41,25 @@ cc = "=1.1.22" cmake = "=0.1.48" build_helper = { path = "../tools/build_helper" } -clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } +build-manifest = { path = "../tools/build-manifest" } +clap = { version = "4.4", default-features = false, features = [ + "std", + "usage", + "help", + "derive", + "error-context", +] } clap_complete = "4.4" fd-lock = "4.0" home = "0.5" ignore = "0.4" libc = "0.2" -object = { version = "0.36.3", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } +object = { version = "0.36.3", default-features = false, features = [ + "archive", + "coff", + "read_core", + "unaligned", +] } opener = "0.5" semver = "1.0" serde = "1.0" @@ -63,7 +75,9 @@ walkdir = "2.4" xz2 = "0.1" # Dependencies needed by the build-metrics feature -sysinfo = { version = "0.31.2", default-features = false, optional = true, features = ["system"] } +sysinfo = { version = "0.31.2", default-features = false, optional = true, features = [ + "system", +] } [target.'cfg(windows)'.dependencies.junction] version = "1.0.0" diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 80ba9f4444863..a967b2fc06d12 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -14,6 +14,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::{env, fs}; +use build_manifest::PkgType; use object::BinaryFormat; use object::read::archive::ArchiveFile; @@ -1432,6 +1433,51 @@ impl Step for Rustfmt { } } +trait WixManifest { + fn wix_name(&self) -> String; + fn wix_group(&self) -> String { + format!("{}Group", self.wix_name()) + } + fn wix_dir_var(&self) -> String { + format!("var.{}Dir", self.wix_name()) + } + fn transform_harvested_output(&self) -> Option<&str>; +} + +impl WixManifest for PkgType { + fn wix_name(&self) -> String { + match self { + PkgType::Rustc => "Rustc", + PkgType::HtmlDocs => "Docs", + PkgType::RustMingw => "Gcc", + PkgType::RustStd => "Std", + PkgType::Cargo => "Cargo", + PkgType::RustAnalysis => "Analysis", + PkgType::RustAnalyzer => "RustAnalyzer", + PkgType::Clippy => "Clippy", + PkgType::Rustfmt => "RustFmt", + PkgType::Miri => "Miri", + _ => { + unimplemented!("wix name not implemented for {:?}", self) + } + } + .to_string() + } + + fn transform_harvested_output(&self) -> Option<&str> { + match self { + PkgType::HtmlDocs => Some("msi/squash-components.xsl"), + PkgType::Cargo + | PkgType::RustAnalysis + | PkgType::RustAnalyzer + | PkgType::Clippy + | PkgType::Rustfmt + | PkgType::Miri => Some("msi/remove-duplicates.xsl"), + _ => None, + } + } +} + #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, @@ -1628,47 +1674,54 @@ impl Step for Extended { } if target.is_windows() { + const CORE_PACKAGES: &[PkgType] = + &[PkgType::Rustc, PkgType::Cargo, PkgType::RustStd, PkgType::RustAnalysis]; + const OPTIONAL_PACKAGES: &[PkgType] = &[ + PkgType::Clippy, + PkgType::Rustfmt, + PkgType::RustAnalyzer, + PkgType::HtmlDocs, + PkgType::Miri, + ]; + + let apply_to_tools = |f: &mut dyn FnMut(&PkgType)| { + for tool in CORE_PACKAGES { + f(tool); + } + for tool in OPTIONAL_PACKAGES { + if built_tools.contains(tool.tarball_component_name()) { + f(tool); + } + } + if target.is_windows_gnu() { + f(&PkgType::RustMingw); + } + }; + let exe = tmp.join("exe"); let _ = fs::remove_dir_all(&exe); - let prepare = |name: &str| { + apply_to_tools(&mut |pkg_type: &PkgType| { + let name = pkg_type.tarball_component_name(); builder.create_dir(&exe.join(name)); - let dir = if name == "rust-std" || name == "rust-analysis" { - format!("{}-{}", name, target.triple) - } else if name == "rust-analyzer" { - "rust-analyzer-preview".to_string() - } else if name == "clippy" { - "clippy-preview".to_string() - } else if name == "rustfmt" { - "rustfmt-preview".to_string() - } else if name == "miri" { - "miri-preview".to_string() - } else if name == "rustc-codegen-cranelift" { - // FIXME add installer support for cg_clif once it is ready to be distributed on - // windows. - unreachable!("cg_clif shouldn't be built for windows"); - } else { - name.to_string() + + let dir = match pkg_type { + PkgType::RustStd | PkgType::RustAnalysis => { + format!("{}-{}", name, target.triple) + } + PkgType::RustcCodegenCranelift => { + // FIXME add installer support for cg_clif once it is ready to be distributed on + // windows. + unreachable!("cg_clif shouldn't be built for windows"); + } + _ => pkg_type.manifest_component_name(), }; builder.cp_link_r( &work.join(format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), &exe.join(name), ); builder.remove(&exe.join(name).join("manifest.in")); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-analysis"); - prepare("rust-std"); - for tool in &["clippy", "rustfmt", "rust-analyzer", "rust-docs", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - if target.is_windows_gnu() { - prepare("rust-mingw"); - } - + }); builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); // Generate msi installer @@ -1680,236 +1733,56 @@ impl Step for Extended { let light = wix.join("bin/light.exe"); let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rustc") - .args(heat_flags) - .arg("-cg") - .arg("RustcGroup") - .arg("-dr") - .arg("Rustc") - .arg("-var") - .arg("var.RustcDir") - .arg("-out") - .arg(exe.join("RustcGroup.wxs")) - .run(builder); - if built_tools.contains("rust-docs") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")) - .run(builder); - } - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("cargo") - .args(heat_flags) - .arg("-cg") - .arg("CargoGroup") - .arg("-dr") - .arg("Cargo") - .arg("-var") - .arg("var.CargoDir") - .arg("-out") - .arg(exe.join("CargoGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-std") - .args(heat_flags) - .arg("-cg") - .arg("StdGroup") - .arg("-dr") - .arg("Std") - .arg("-var") - .arg("var.StdDir") - .arg("-out") - .arg(exe.join("StdGroup.wxs")) - .run(builder); - if built_tools.contains("rust-analyzer") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analyzer") - .args(heat_flags) - .arg("-cg") - .arg("RustAnalyzerGroup") - .arg("-dr") - .arg("RustAnalyzer") - .arg("-var") - .arg("var.RustAnalyzerDir") - .arg("-out") - .arg(exe.join("RustAnalyzerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - if built_tools.contains("clippy") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("clippy") - .args(heat_flags) - .arg("-cg") - .arg("ClippyGroup") - .arg("-dr") - .arg("Clippy") - .arg("-var") - .arg("var.ClippyDir") - .arg("-out") - .arg(exe.join("ClippyGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - if built_tools.contains("rustfmt") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rustfmt") - .args(heat_flags) - .arg("-cg") - .arg("RustFmtGroup") - .arg("-dr") - .arg("RustFmt") - .arg("-var") - .arg("var.RustFmtDir") - .arg("-out") - .arg(exe.join("RustFmtGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - if built_tools.contains("miri") { - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("miri") - .args(heat_flags) - .arg("-cg") - .arg("MiriGroup") - .arg("-dr") - .arg("Miri") - .arg("-var") - .arg("var.MiriDir") - .arg("-out") - .arg(exe.join("MiriGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - } - command(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analysis") - .args(heat_flags) - .arg("-cg") - .arg("AnalysisGroup") - .arg("-dr") - .arg("Analysis") - .arg("-var") - .arg("var.AnalysisDir") - .arg("-out") - .arg(exe.join("AnalysisGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")) - .run(builder); - if target.is_windows_gnu() { - command(&heat) + + apply_to_tools(&mut |pkg_type: &PkgType| { + let mut binding = command(&heat); + let cmd = binding .current_dir(&exe) .arg("dir") - .arg("rust-mingw") + .arg(pkg_type.tarball_component_name()) .args(heat_flags) .arg("-cg") - .arg("GccGroup") + .arg(pkg_type.wix_group()) .arg("-dr") - .arg("Gcc") + .arg(pkg_type.wix_name()) .arg("-var") - .arg("var.GccDir") + .arg(pkg_type.wix_dir_var()) .arg("-out") - .arg(exe.join("GccGroup.wxs")) - .run(builder); - } + .arg(exe.join(format!("{}.wxs", pkg_type.wix_group()))); + if let Some(t) = pkg_type.transform_harvested_output() { + cmd.arg("-t").arg(etc.join(t)); + } + cmd.run(builder); + }); + fn candle_arg(pkg_type: &PkgType) -> String { + format!("-d{}Dir={}", pkg_type.wix_name(), pkg_type.tarball_component_name()) + } let candle = |input: &Path| { let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); let arch = if target.contains("x86_64") { "x64" } else { "x86" }; let mut cmd = command(&candle); cmd.current_dir(&exe) .arg("-nologo") - .arg("-dRustcDir=rustc") - .arg("-dCargoDir=cargo") - .arg("-dStdDir=rust-std") - .arg("-dAnalysisDir=rust-analysis") .arg("-arch") .arg(arch) .arg("-out") .arg(&output) .arg(input); - add_env(builder, &mut cmd, target); - if built_tools.contains("clippy") { - cmd.arg("-dClippyDir=clippy"); - } - if built_tools.contains("rustfmt") { - cmd.arg("-dRustFmtDir=rustfmt"); - } - if built_tools.contains("rust-docs") { - cmd.arg("-dDocsDir=rust-docs"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("-dRustAnalyzerDir=rust-analyzer"); - } - if built_tools.contains("miri") { - cmd.arg("-dMiriDir=miri"); - } - if target.is_windows_gnu() { - cmd.arg("-dGccDir=rust-mingw"); - } + add_env(builder, &mut cmd, target); + apply_to_tools(&mut |pkg_type: &PkgType| { + cmd.arg(candle_arg(pkg_type)); + }); cmd.run(builder); }; candle(&xform(&etc.join("msi/rust.wxs"))); candle(&etc.join("msi/ui.wxs")); candle(&etc.join("msi/rustwelcomedlg.wxs")); - candle("RustcGroup.wxs".as_ref()); - if built_tools.contains("rust-docs") { - candle("DocsGroup.wxs".as_ref()); - } - candle("CargoGroup.wxs".as_ref()); - candle("StdGroup.wxs".as_ref()); - if built_tools.contains("clippy") { - candle("ClippyGroup.wxs".as_ref()); - } - if built_tools.contains("rustfmt") { - candle("RustFmtGroup.wxs".as_ref()); - } - if built_tools.contains("miri") { - candle("MiriGroup.wxs".as_ref()); - } - if built_tools.contains("rust-analyzer") { - candle("RustAnalyzerGroup.wxs".as_ref()); - } - candle("AnalysisGroup.wxs".as_ref()); - if target.is_windows_gnu() { - candle("GccGroup.wxs".as_ref()); - } + apply_to_tools(&mut |pkg_type: &PkgType| { + candle(format!("{}.wxs", pkg_type.wix_group()).as_ref()) + }); builder.create(&exe.join("LICENSE.rtf"), &rtf); builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); @@ -1927,32 +1800,13 @@ impl Step for Extended { .arg(exe.join(&filename)) .arg("rust.wixobj") .arg("ui.wixobj") - .arg("rustwelcomedlg.wixobj") - .arg("RustcGroup.wixobj") - .arg("CargoGroup.wixobj") - .arg("StdGroup.wixobj") - .arg("AnalysisGroup.wixobj") - .current_dir(&exe); - - if built_tools.contains("clippy") { - cmd.arg("ClippyGroup.wixobj"); - } - if built_tools.contains("rustfmt") { - cmd.arg("RustFmtGroup.wixobj"); - } - if built_tools.contains("miri") { - cmd.arg("MiriGroup.wixobj"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("RustAnalyzerGroup.wixobj"); - } - if built_tools.contains("rust-docs") { - cmd.arg("DocsGroup.wixobj"); - } + .arg("rustwelcomedlg.wixobj"); - if target.is_windows_gnu() { - cmd.arg("GccGroup.wixobj"); - } + apply_to_tools(&mut |pkg_type: &PkgType| { + cmd.arg(format!("{}.wixobj", pkg_type.wix_group())); + }); + + cmd.current_dir(&exe); // ICE57 wrongly complains about the shortcuts cmd.arg("-sice:ICE57"); diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 07460b81412ac..16a7e8380194c 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -16,7 +16,6 @@ use std::{cmp, env, fs}; use build_helper::exit; use build_helper::git::{GitConfig, get_closest_merge_commit, output_result}; use serde::{Deserialize, Deserializer}; -use serde_derive::Deserialize; use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; use crate::core::build_steps::llvm; diff --git a/src/etc/installer/msi/rust.wxs b/src/etc/installer/msi/rust.wxs index 2d155bf0b1019..63811ad4622cb 100644 --- a/src/etc/installer/msi/rust.wxs +++ b/src/etc/installer/msi/rust.wxs @@ -285,35 +285,35 @@ diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 2ea1bffb35f4d..35aaafec1b3da 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -9,7 +9,7 @@ This gets called by `promote-release` { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + ($e:expr, $extra:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}: {}", stringify!($e), e, $extra), + } + }; +} + +pub struct Builder { + pub versions: Versions, + pub checksums: Checksums, + pub shipped_files: HashSet, + + pub input: PathBuf, + pub output: PathBuf, + pub s3_address: String, + pub date: String, +} + +impl Builder { + pub fn build(&mut self) { + let manifest = self.build_manifest(); + + let channel = self.versions.channel().to_string(); + self.write_channel_files(&channel, &manifest); + if channel == "stable" { + // channel-rust-1.XX.YY.toml + let rust_version = self.versions.rustc_version().to_string(); + self.write_channel_files(&rust_version, &manifest); + + // channel-rust-1.XX.toml + let major_minor = rust_version.split('.').take(2).collect::>().join("."); + self.write_channel_files(&major_minor, &manifest); + } else if channel == "beta" { + // channel-rust-1.XX.YY-beta.Z.toml + let rust_version = self + .versions + .version(&PkgType::Rust) + .expect("missing Rust tarball") + .version + .expect("missing Rust version") + .split(' ') + .next() + .unwrap() + .to_string(); + self.write_channel_files(&rust_version, &manifest); + + // channel-rust-1.XX.YY-beta.toml + let major_minor_patch_beta = + rust_version.split('.').take(3).collect::>().join("."); + self.write_channel_files(&major_minor_patch_beta, &manifest); + + // channel-rust-1.XX-beta.toml + let major_minor_beta = + format!("{}-beta", rust_version.split('.').take(2).collect::>().join(".")); + self.write_channel_files(&major_minor_beta, &manifest); + } + + if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") { + self.write_shipped_files(&Path::new(&path)); + } + + t!(self.checksums.store_cache()); + } + + fn build_manifest(&mut self) -> Manifest { + let mut manifest = Manifest { + manifest_version: "2".to_string(), + date: self.date.to_string(), + pkg: BTreeMap::new(), + artifacts: BTreeMap::new(), + renames: BTreeMap::new(), + profiles: BTreeMap::new(), + }; + self.add_packages_to(&mut manifest); + self.add_artifacts_to(&mut manifest); + self.add_profiles_to(&mut manifest); + self.add_renames_to(&mut manifest); + manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest)); + + self.checksums.fill_missing_checksums(&mut manifest); + + manifest + } + + fn add_packages_to(&mut self, manifest: &mut Manifest) { + for pkg in PkgType::all() { + self.package(pkg, &mut manifest.pkg); + } + } + + fn add_artifacts_to(&mut self, manifest: &mut Manifest) { + manifest.add_artifact("source-code", |artifact| { + let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap(); + artifact.add_tarball(self, "*", &tarball); + }); + + manifest.add_artifact("installer-msi", |artifact| { + for target in MSI_INSTALLERS { + let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap(); + artifact.add_file(self, target, &msi); + } + }); + + manifest.add_artifact("installer-pkg", |artifact| { + for target in PKG_INSTALLERS { + let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap(); + artifact.add_file(self, target, &pkg); + } + }); + } + + fn add_profiles_to(&mut self, manifest: &mut Manifest) { + use PkgType::*; + + let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs); + + // Use a Vec here to make sure we don't exclude any components in an earlier profile. + let minimal = vec![Rustc, Cargo, RustStd, RustMingw]; + profile("minimal", &minimal); + + let mut default = minimal; + default.extend([HtmlDocs, Rustfmt, Clippy]); + profile("default", &default); + + // NOTE: this profile is effectively deprecated; do not add new components to it. + let mut complete = default; + complete.extend([ + Rls, + RustAnalyzer, + RustSrc, + LlvmTools, + RustAnalysis, + Miri, + RustcCodegenCranelift, + ]); + profile("complete", &complete); + + // The compiler libraries are not stable for end users, and they're also huge, so we only + // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible + // for users to install the additional component manually, if needed. + if self.versions.channel() == "nightly" { + self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]); + // Do not include the rustc-docs component for now, as it causes + // conflicts with the rust-docs component when installed. See + // #75833. + // self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]); + } + } + + fn add_renames_to(&self, manifest: &mut Manifest) { + let mut rename = |from: &str, to: &str| { + manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() }) + }; + for pkg in PkgType::all() { + if pkg.is_preview() { + rename(pkg.tarball_component_name(), &pkg.manifest_component_name()); + } + } + } + + fn rust_package(&mut self, manifest: &Manifest) -> Package { + let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball"); + let mut pkg = Package { + version: version_info.version.expect("missing Rust version"), + git_commit_hash: version_info.git_commit, + target: BTreeMap::new(), + }; + for host in HOSTS { + if let Some(target) = self.target_host_combination(host, &manifest) { + pkg.target.insert(host.to_string(), target); + } else { + pkg.target.insert(host.to_string(), Target::unavailable()); + continue; + } + } + pkg + } + + fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option { + let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap(); + + let mut target = Target::from_compressed_tar(self, &filename); + if !target.available { + return None; + } + + let mut components = Vec::new(); + let mut extensions = Vec::new(); + + let host_component = |pkg: &_| Component::from_pkg(pkg, host); + + for pkg in PkgType::all() { + match pkg { + // rustc/rust-std/cargo/docs are all required + PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => { + components.push(host_component(pkg)); + } + PkgType::RustStd => { + components.push(host_component(pkg)); + extensions.extend( + TARGETS + .iter() + .filter(|&&target| target != host) + .map(|target| Component::from_pkg(pkg, target)), + ); + } + // so is rust-mingw if it's available for the target + PkgType::RustMingw => { + if host.contains("pc-windows-gnu") { + components.push(host_component(pkg)); + } + } + // Tools are always present in the manifest, + // but might be marked as unavailable if they weren't built. + PkgType::Clippy + | PkgType::Miri + | PkgType::Rls + | PkgType::RustAnalyzer + | PkgType::Rustfmt + | PkgType::LlvmTools + | PkgType::RustAnalysis + | PkgType::JsonDocs + | PkgType::RustcCodegenCranelift + | PkgType::LlvmBitcodeLinker => { + extensions.push(host_component(pkg)); + } + PkgType::RustcDev | PkgType::RustcDocs => { + extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target))); + } + PkgType::RustSrc => { + extensions.push(Component::from_pkg(pkg, "*")); + } + PkgType::Rust => {} + // NOTE: this is intentional, these artifacts aren't intended to be used with rustup + PkgType::ReproducibleArtifacts => {} + } + } + + // If the components/extensions don't actually exist for this + // particular host/target combination then nix it entirely from our + // lists. + let has_component = |c: &Component| { + if c.target == "*" { + return true; + } + let pkg = match manifest.pkg.get(&c.pkg) { + Some(p) => p, + None => return false, + }; + pkg.target.contains_key(&c.target) + }; + extensions.retain(&has_component); + components.retain(&has_component); + + target.components = Some(components); + target.extensions = Some(extensions); + Some(target) + } + + fn profile( + &mut self, + profile_name: &str, + dst: &mut BTreeMap>, + pkgs: &[PkgType], + ) { + dst.insert( + profile_name.to_owned(), + pkgs.iter().map(|s| s.manifest_component_name()).collect(), + ); + } + + fn extend_profile( + &mut self, + profile_name: &str, + dst: &mut BTreeMap>, + pkgs: &[PkgType], + ) { + dst.get_mut(profile_name) + .expect("existing profile") + .extend(pkgs.iter().map(|s| s.manifest_component_name())); + } + + fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap) { + if *pkg == PkgType::Rust { + // This is handled specially by `rust_package` later. + // Order is important, so don't call `rust_package` here. + return; + } + + let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; + let version_info = self.versions.version(&pkg).expect("failed to load package version"); + let mut is_present = version_info.present; + + // Never ship nightly-only components for other trains. + if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) { + is_present = false; // Pretend the component is entirely missing. + } + + macro_rules! tarball_name { + ($target_name:expr) => { + self.versions.tarball_name(pkg, $target_name).unwrap() + }; + } + let mut target_from_compressed_tar = |target_name| { + let target = Target::from_compressed_tar(self, &tarball_name!(target_name)); + if target.available { + return target; + } + for (substr, fallback_target) in fallback { + if target_name.contains(substr) { + let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target)); + // Fallbacks should typically be available on 'production' builds + // but may not be available for try builds, which only build one target by + // default. Ideally we'd gate this being a hard error on whether we're in a + // production build or not, but it's not information that's readily available + // here. + if !t.available { + eprintln!( + "{:?} not available for fallback", + tarball_name!(fallback_target) + ); + continue; + } + return t; + } + } + Target::unavailable() + }; + + let targets = pkg + .targets() + .iter() + .map(|name| { + let target = if is_present { + target_from_compressed_tar(name) + } else { + // If the component is not present for this build add it anyway but mark it as + // unavailable -- this way rustup won't allow upgrades without --force + Target::unavailable() + }; + (name.to_string(), target) + }) + .collect(); + + dst.insert(pkg.manifest_component_name(), Package { + version: version_info.version.unwrap_or_default(), + git_commit_hash: version_info.git_commit, + target: targets, + }); + } + + pub(crate) fn url(&self, path: &Path) -> String { + let file_name = path.file_name().unwrap().to_str().unwrap(); + format!("{}/{}/{}", self.s3_address, self.date, file_name) + } + + fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) { + self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml"); + self.write(&manifest.date, channel_name, "-date.txt"); + self.write( + manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(), + channel_name, + "-git-commit-hash.txt", + ); + } + + fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) { + let name = format!("channel-rust-{}{}", channel_name, suffix); + self.shipped_files.insert(name.clone()); + + let dst = self.output.join(name); + t!(fs::write(&dst, contents), format!("failed to create manifest {}", dst.display())); + } + + fn write_shipped_files(&self, path: &Path) { + let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::>(); + files.sort(); + let content = format!("{}\n", files.join("\n")); + + t!(std::fs::write(path, content.as_bytes())); + } +} diff --git a/src/tools/build-manifest/src/checksum.rs b/src/tools/build-manifest/src/checksum.rs index c69b4180d92de..03dc1f28e4f6a 100644 --- a/src/tools/build-manifest/src/checksum.rs +++ b/src/tools/build-manifest/src/checksum.rs @@ -11,13 +11,13 @@ use sha2::{Digest, Sha256}; use crate::manifest::{FileHash, Manifest}; -pub(crate) struct Checksums { +pub struct Checksums { cache_path: Option, collected: Mutex>, } impl Checksums { - pub(crate) fn new() -> Result> { + pub fn new() -> Result> { let cache_path = std::env::var_os("BUILD_MANIFEST_CHECKSUM_CACHE").map(PathBuf::from); let mut collected = HashMap::new(); diff --git a/src/tools/build-manifest/src/constants.rs b/src/tools/build-manifest/src/constants.rs new file mode 100644 index 0000000000000..8145a79f4d828 --- /dev/null +++ b/src/tools/build-manifest/src/constants.rs @@ -0,0 +1,204 @@ +use crate::versions::PkgType; + +pub(crate) static HOSTS: &[&str] = &[ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "i686-apple-darwin", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "loongarch64-unknown-linux-gnu", + "loongarch64-unknown-linux-musl", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "mipsisa32r6-unknown-linux-gnu", + "mipsisa32r6el-unknown-linux-gnu", + "mipsisa64r6-unknown-linux-gnuabi64", + "mipsisa64r6el-unknown-linux-gnuabi64", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", +]; + +pub(crate) static TARGETS: &[&str] = &[ + "aarch64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-macabi", + "aarch64-apple-ios-sim", + "aarch64-unknown-fuchsia", + "aarch64-linux-android", + "aarch64-pc-windows-gnullvm", + "aarch64-pc-windows-msvc", + "aarch64-unknown-hermit", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-unknown-linux-ohos", + "aarch64-unknown-none", + "aarch64-unknown-none-softfloat", + "aarch64-unknown-redox", + "aarch64-unknown-uefi", + "arm64e-apple-darwin", + "arm64e-apple-ios", + "arm64e-apple-tvos", + "arm-linux-androideabi", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "arm-unknown-linux-musleabi", + "arm-unknown-linux-musleabihf", + "arm64ec-pc-windows-msvc", + "armv5te-unknown-linux-gnueabi", + "armv5te-unknown-linux-musleabi", + "armv7-linux-androideabi", + "thumbv7neon-linux-androideabi", + "armv7-unknown-linux-gnueabi", + "armv7-unknown-linux-gnueabihf", + "armv7a-none-eabi", + "thumbv7neon-unknown-linux-gnueabihf", + "armv7-unknown-linux-musleabi", + "armv7-unknown-linux-musleabihf", + "armv7-unknown-linux-ohos", + "armebv7r-none-eabi", + "armebv7r-none-eabihf", + "armv7r-none-eabi", + "armv7r-none-eabihf", + "armv8r-none-eabihf", + "armv7s-apple-ios", + "bpfeb-unknown-none", + "bpfel-unknown-none", + "i386-apple-ios", + "i586-pc-windows-msvc", + "i586-unknown-linux-gnu", + "i586-unknown-linux-musl", + "i686-apple-darwin", + "i686-linux-android", + "i686-pc-windows-gnu", + "i686-pc-windows-gnullvm", + "i686-pc-windows-msvc", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "i686-unknown-linux-musl", + "i686-unknown-redox", + "i686-unknown-uefi", + "loongarch64-unknown-linux-gnu", + "loongarch64-unknown-linux-musl", + "loongarch64-unknown-none", + "loongarch64-unknown-none-softfloat", + "m68k-unknown-linux-gnu", + "csky-unknown-linux-gnuabiv2", + "csky-unknown-linux-gnuabiv2hf", + "mips-unknown-linux-gnu", + "mips-unknown-linux-musl", + "mips64-unknown-linux-gnuabi64", + "mips64-unknown-linux-muslabi64", + "mips64el-unknown-linux-gnuabi64", + "mips64el-unknown-linux-muslabi64", + "mipsisa32r6-unknown-linux-gnu", + "mipsisa32r6el-unknown-linux-gnu", + "mipsisa64r6-unknown-linux-gnuabi64", + "mipsisa64r6el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "mipsel-unknown-linux-musl", + "nvptx64-nvidia-cuda", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv32i-unknown-none-elf", + "riscv32im-risc0-zkvm-elf", + "riscv32im-unknown-none-elf", + "riscv32ima-unknown-none-elf", + "riscv32imc-unknown-none-elf", + "riscv32imac-unknown-none-elf", + "riscv32imafc-unknown-none-elf", + "riscv32gc-unknown-linux-gnu", + "riscv64imac-unknown-none-elf", + "riscv64gc-unknown-hermit", + "riscv64gc-unknown-none-elf", + "riscv64gc-unknown-linux-gnu", + "riscv64gc-unknown-linux-musl", + "s390x-unknown-linux-gnu", + "sparc64-unknown-linux-gnu", + "sparcv9-sun-solaris", + "sparc-unknown-none-elf", + "thumbv6m-none-eabi", + "thumbv7em-none-eabi", + "thumbv7em-none-eabihf", + "thumbv7m-none-eabi", + "thumbv8m.base-none-eabi", + "thumbv8m.main-none-eabi", + "thumbv8m.main-none-eabihf", + "wasm32-unknown-emscripten", + "wasm32-unknown-unknown", + "wasm32-wasi", + "wasm32-wasip1", + "wasm32-wasip1-threads", + "wasm32-wasip2", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-apple-ios-macabi", + "x86_64-fortanix-unknown-sgx", + "x86_64-unknown-fuchsia", + "x86_64-linux-android", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-gnullvm", + "x86_64-pc-windows-msvc", + "x86_64-pc-solaris", + "x86_64-unikraft-linux-musl", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-gnux32", + "x86_64-unknown-linux-musl", + "x86_64-unknown-linux-ohos", + "x86_64-unknown-netbsd", + "x86_64-unknown-none", + "x86_64-unknown-redox", + "x86_64-unknown-hermit", + "x86_64-unknown-uefi", +]; + +/// This allows the manifest to contain rust-docs for hosts that don't build +/// docs. +/// +/// Tuples of `(host_partial, host_instead)`. If the host does not have the +/// rust-docs component available, then if the host name contains +/// `host_partial`, it will use the docs from `host_instead` instead. +/// +/// The order here matters, more specific entries should be first. +pub(crate) static DOCS_FALLBACK: &[(&str, &str)] = &[ + ("-apple-", "x86_64-apple-darwin"), + ("aarch64", "aarch64-unknown-linux-gnu"), + ("arm-", "aarch64-unknown-linux-gnu"), + ("", "x86_64-unknown-linux-gnu"), +]; + +pub(crate) static MSI_INSTALLERS: &[&str] = &[ + "aarch64-pc-windows-msvc", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", +]; + +pub(crate) static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"]; + +pub(crate) static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; + +pub(crate) static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = + &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift]; diff --git a/src/tools/build-manifest/src/lib.rs b/src/tools/build-manifest/src/lib.rs new file mode 100644 index 0000000000000..476b7579d4035 --- /dev/null +++ b/src/tools/build-manifest/src/lib.rs @@ -0,0 +1,10 @@ +mod builder; +mod checksum; +mod constants; +mod manifest; +mod versions; + +pub use builder::Builder; +pub use checksum::Checksums; +pub(crate) use constants::*; +pub use versions::{PkgType, Versions}; diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 62e1695cbe362..c3d9076ae6470 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -1,245 +1,10 @@ #![doc = include_str!("../README.md")] -mod checksum; -mod manifest; -mod versions; +use std::collections::HashSet; +use std::env; +use std::path::PathBuf; -use std::collections::{BTreeMap, HashSet}; -use std::path::{Path, PathBuf}; -use std::{env, fs}; - -use crate::checksum::Checksums; -use crate::manifest::{Component, Manifest, Package, Rename, Target}; -use crate::versions::{PkgType, Versions}; - -static HOSTS: &[&str] = &[ - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "i686-apple-darwin", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "loongarch64-unknown-linux-gnu", - "loongarch64-unknown-linux-musl", - "mips-unknown-linux-gnu", - "mips64-unknown-linux-gnuabi64", - "mips64el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "mipsisa32r6-unknown-linux-gnu", - "mipsisa32r6el-unknown-linux-gnu", - "mipsisa64r6-unknown-linux-gnuabi64", - "mipsisa64r6el-unknown-linux-gnuabi64", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", -]; - -static TARGETS: &[&str] = &[ - "aarch64-apple-darwin", - "aarch64-apple-ios", - "aarch64-apple-ios-macabi", - "aarch64-apple-ios-sim", - "aarch64-unknown-fuchsia", - "aarch64-linux-android", - "aarch64-pc-windows-gnullvm", - "aarch64-pc-windows-msvc", - "aarch64-unknown-hermit", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "aarch64-unknown-linux-ohos", - "aarch64-unknown-none", - "aarch64-unknown-none-softfloat", - "aarch64-unknown-redox", - "aarch64-unknown-uefi", - "arm64e-apple-darwin", - "arm64e-apple-ios", - "arm64e-apple-tvos", - "arm-linux-androideabi", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "arm-unknown-linux-musleabi", - "arm-unknown-linux-musleabihf", - "arm64ec-pc-windows-msvc", - "armv5te-unknown-linux-gnueabi", - "armv5te-unknown-linux-musleabi", - "armv7-linux-androideabi", - "thumbv7neon-linux-androideabi", - "armv7-unknown-linux-gnueabi", - "armv7-unknown-linux-gnueabihf", - "armv7a-none-eabi", - "thumbv7neon-unknown-linux-gnueabihf", - "armv7-unknown-linux-musleabi", - "armv7-unknown-linux-musleabihf", - "armv7-unknown-linux-ohos", - "armebv7r-none-eabi", - "armebv7r-none-eabihf", - "armv7r-none-eabi", - "armv7r-none-eabihf", - "armv8r-none-eabihf", - "armv7s-apple-ios", - "bpfeb-unknown-none", - "bpfel-unknown-none", - "i386-apple-ios", - "i586-pc-windows-msvc", - "i586-unknown-linux-gnu", - "i586-unknown-linux-musl", - "i686-apple-darwin", - "i686-linux-android", - "i686-pc-windows-gnu", - "i686-pc-windows-gnullvm", - "i686-pc-windows-msvc", - "i686-unknown-freebsd", - "i686-unknown-linux-gnu", - "i686-unknown-linux-musl", - "i686-unknown-redox", - "i686-unknown-uefi", - "loongarch64-unknown-linux-gnu", - "loongarch64-unknown-linux-musl", - "loongarch64-unknown-none", - "loongarch64-unknown-none-softfloat", - "m68k-unknown-linux-gnu", - "csky-unknown-linux-gnuabiv2", - "csky-unknown-linux-gnuabiv2hf", - "mips-unknown-linux-gnu", - "mips-unknown-linux-musl", - "mips64-unknown-linux-gnuabi64", - "mips64-unknown-linux-muslabi64", - "mips64el-unknown-linux-gnuabi64", - "mips64el-unknown-linux-muslabi64", - "mipsisa32r6-unknown-linux-gnu", - "mipsisa32r6el-unknown-linux-gnu", - "mipsisa64r6-unknown-linux-gnuabi64", - "mipsisa64r6el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "mipsel-unknown-linux-musl", - "nvptx64-nvidia-cuda", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv32i-unknown-none-elf", - "riscv32im-risc0-zkvm-elf", - "riscv32im-unknown-none-elf", - "riscv32ima-unknown-none-elf", - "riscv32imc-unknown-none-elf", - "riscv32imac-unknown-none-elf", - "riscv32imafc-unknown-none-elf", - "riscv32gc-unknown-linux-gnu", - "riscv64imac-unknown-none-elf", - "riscv64gc-unknown-hermit", - "riscv64gc-unknown-none-elf", - "riscv64gc-unknown-linux-gnu", - "riscv64gc-unknown-linux-musl", - "s390x-unknown-linux-gnu", - "sparc64-unknown-linux-gnu", - "sparcv9-sun-solaris", - "sparc-unknown-none-elf", - "thumbv6m-none-eabi", - "thumbv7em-none-eabi", - "thumbv7em-none-eabihf", - "thumbv7m-none-eabi", - "thumbv8m.base-none-eabi", - "thumbv8m.main-none-eabi", - "thumbv8m.main-none-eabihf", - "wasm32-unknown-emscripten", - "wasm32-unknown-unknown", - "wasm32-wasi", - "wasm32-wasip1", - "wasm32-wasip1-threads", - "wasm32-wasip2", - "x86_64-apple-darwin", - "x86_64-apple-ios", - "x86_64-apple-ios-macabi", - "x86_64-fortanix-unknown-sgx", - "x86_64-unknown-fuchsia", - "x86_64-linux-android", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-gnullvm", - "x86_64-pc-windows-msvc", - "x86_64-pc-solaris", - "x86_64-unikraft-linux-musl", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-gnux32", - "x86_64-unknown-linux-musl", - "x86_64-unknown-linux-ohos", - "x86_64-unknown-netbsd", - "x86_64-unknown-none", - "x86_64-unknown-redox", - "x86_64-unknown-hermit", - "x86_64-unknown-uefi", -]; - -/// This allows the manifest to contain rust-docs for hosts that don't build -/// docs. -/// -/// Tuples of `(host_partial, host_instead)`. If the host does not have the -/// rust-docs component available, then if the host name contains -/// `host_partial`, it will use the docs from `host_instead` instead. -/// -/// The order here matters, more specific entries should be first. -static DOCS_FALLBACK: &[(&str, &str)] = &[ - ("-apple-", "x86_64-apple-darwin"), - ("aarch64", "aarch64-unknown-linux-gnu"), - ("arm-", "aarch64-unknown-linux-gnu"), - ("", "x86_64-unknown-linux-gnu"), -]; - -static MSI_INSTALLERS: &[&str] = &[ - "aarch64-pc-windows-msvc", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", -]; - -static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"]; - -static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"]; - -static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = - &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift]; - -macro_rules! t { - ($e:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - } - }; - ($e:expr, $extra:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}: {}", stringify!($e), e, $extra), - } - }; -} - -struct Builder { - versions: Versions, - checksums: Checksums, - shipped_files: HashSet, - - input: PathBuf, - output: PathBuf, - s3_address: String, - date: String, -} +use build_manifest::{Builder, Checksums, Versions}; fn main() { let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") { @@ -261,7 +26,7 @@ fn main() { Builder { versions: Versions::new(&channel, &input).unwrap(), - checksums: t!(Checksums::new()), + checksums: build_manifest::t!(Checksums::new()), shipped_files: HashSet::new(), input, @@ -271,368 +36,3 @@ fn main() { } .build(); } - -impl Builder { - fn build(&mut self) { - let manifest = self.build_manifest(); - - let channel = self.versions.channel().to_string(); - self.write_channel_files(&channel, &manifest); - if channel == "stable" { - // channel-rust-1.XX.YY.toml - let rust_version = self.versions.rustc_version().to_string(); - self.write_channel_files(&rust_version, &manifest); - - // channel-rust-1.XX.toml - let major_minor = rust_version.split('.').take(2).collect::>().join("."); - self.write_channel_files(&major_minor, &manifest); - } else if channel == "beta" { - // channel-rust-1.XX.YY-beta.Z.toml - let rust_version = self - .versions - .version(&PkgType::Rust) - .expect("missing Rust tarball") - .version - .expect("missing Rust version") - .split(' ') - .next() - .unwrap() - .to_string(); - self.write_channel_files(&rust_version, &manifest); - - // channel-rust-1.XX.YY-beta.toml - let major_minor_patch_beta = - rust_version.split('.').take(3).collect::>().join("."); - self.write_channel_files(&major_minor_patch_beta, &manifest); - - // channel-rust-1.XX-beta.toml - let major_minor_beta = - format!("{}-beta", rust_version.split('.').take(2).collect::>().join(".")); - self.write_channel_files(&major_minor_beta, &manifest); - } - - if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") { - self.write_shipped_files(&Path::new(&path)); - } - - t!(self.checksums.store_cache()); - } - - fn build_manifest(&mut self) -> Manifest { - let mut manifest = Manifest { - manifest_version: "2".to_string(), - date: self.date.to_string(), - pkg: BTreeMap::new(), - artifacts: BTreeMap::new(), - renames: BTreeMap::new(), - profiles: BTreeMap::new(), - }; - self.add_packages_to(&mut manifest); - self.add_artifacts_to(&mut manifest); - self.add_profiles_to(&mut manifest); - self.add_renames_to(&mut manifest); - manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest)); - - self.checksums.fill_missing_checksums(&mut manifest); - - manifest - } - - fn add_packages_to(&mut self, manifest: &mut Manifest) { - for pkg in PkgType::all() { - self.package(pkg, &mut manifest.pkg); - } - } - - fn add_artifacts_to(&mut self, manifest: &mut Manifest) { - manifest.add_artifact("source-code", |artifact| { - let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap(); - artifact.add_tarball(self, "*", &tarball); - }); - - manifest.add_artifact("installer-msi", |artifact| { - for target in MSI_INSTALLERS { - let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap(); - artifact.add_file(self, target, &msi); - } - }); - - manifest.add_artifact("installer-pkg", |artifact| { - for target in PKG_INSTALLERS { - let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap(); - artifact.add_file(self, target, &pkg); - } - }); - } - - fn add_profiles_to(&mut self, manifest: &mut Manifest) { - use PkgType::*; - - let mut profile = |name, pkgs: &_| self.profile(name, &mut manifest.profiles, pkgs); - - // Use a Vec here to make sure we don't exclude any components in an earlier profile. - let minimal = vec![Rustc, Cargo, RustStd, RustMingw]; - profile("minimal", &minimal); - - let mut default = minimal; - default.extend([HtmlDocs, Rustfmt, Clippy]); - profile("default", &default); - - // NOTE: this profile is effectively deprecated; do not add new components to it. - let mut complete = default; - complete.extend([ - Rls, - RustAnalyzer, - RustSrc, - LlvmTools, - RustAnalysis, - Miri, - RustcCodegenCranelift, - ]); - profile("complete", &complete); - - // The compiler libraries are not stable for end users, and they're also huge, so we only - // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible - // for users to install the additional component manually, if needed. - if self.versions.channel() == "nightly" { - self.extend_profile("complete", &mut manifest.profiles, &[RustcDev]); - // Do not include the rustc-docs component for now, as it causes - // conflicts with the rust-docs component when installed. See - // #75833. - // self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]); - } - } - - fn add_renames_to(&self, manifest: &mut Manifest) { - let mut rename = |from: &str, to: &str| { - manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() }) - }; - for pkg in PkgType::all() { - if pkg.is_preview() { - rename(pkg.tarball_component_name(), &pkg.manifest_component_name()); - } - } - } - - fn rust_package(&mut self, manifest: &Manifest) -> Package { - let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball"); - let mut pkg = Package { - version: version_info.version.expect("missing Rust version"), - git_commit_hash: version_info.git_commit, - target: BTreeMap::new(), - }; - for host in HOSTS { - if let Some(target) = self.target_host_combination(host, &manifest) { - pkg.target.insert(host.to_string(), target); - } else { - pkg.target.insert(host.to_string(), Target::unavailable()); - continue; - } - } - pkg - } - - fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option { - let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap(); - - let mut target = Target::from_compressed_tar(self, &filename); - if !target.available { - return None; - } - - let mut components = Vec::new(); - let mut extensions = Vec::new(); - - let host_component = |pkg: &_| Component::from_pkg(pkg, host); - - for pkg in PkgType::all() { - match pkg { - // rustc/rust-std/cargo/docs are all required - PkgType::Rustc | PkgType::Cargo | PkgType::HtmlDocs => { - components.push(host_component(pkg)); - } - PkgType::RustStd => { - components.push(host_component(pkg)); - extensions.extend( - TARGETS - .iter() - .filter(|&&target| target != host) - .map(|target| Component::from_pkg(pkg, target)), - ); - } - // so is rust-mingw if it's available for the target - PkgType::RustMingw => { - if host.contains("pc-windows-gnu") { - components.push(host_component(pkg)); - } - } - // Tools are always present in the manifest, - // but might be marked as unavailable if they weren't built. - PkgType::Clippy - | PkgType::Miri - | PkgType::Rls - | PkgType::RustAnalyzer - | PkgType::Rustfmt - | PkgType::LlvmTools - | PkgType::RustAnalysis - | PkgType::JsonDocs - | PkgType::RustcCodegenCranelift - | PkgType::LlvmBitcodeLinker => { - extensions.push(host_component(pkg)); - } - PkgType::RustcDev | PkgType::RustcDocs => { - extensions.extend(HOSTS.iter().map(|target| Component::from_pkg(pkg, target))); - } - PkgType::RustSrc => { - extensions.push(Component::from_pkg(pkg, "*")); - } - PkgType::Rust => {} - // NOTE: this is intentional, these artifacts aren't intended to be used with rustup - PkgType::ReproducibleArtifacts => {} - } - } - - // If the components/extensions don't actually exist for this - // particular host/target combination then nix it entirely from our - // lists. - let has_component = |c: &Component| { - if c.target == "*" { - return true; - } - let pkg = match manifest.pkg.get(&c.pkg) { - Some(p) => p, - None => return false, - }; - pkg.target.contains_key(&c.target) - }; - extensions.retain(&has_component); - components.retain(&has_component); - - target.components = Some(components); - target.extensions = Some(extensions); - Some(target) - } - - fn profile( - &mut self, - profile_name: &str, - dst: &mut BTreeMap>, - pkgs: &[PkgType], - ) { - dst.insert( - profile_name.to_owned(), - pkgs.iter().map(|s| s.manifest_component_name()).collect(), - ); - } - - fn extend_profile( - &mut self, - profile_name: &str, - dst: &mut BTreeMap>, - pkgs: &[PkgType], - ) { - dst.get_mut(profile_name) - .expect("existing profile") - .extend(pkgs.iter().map(|s| s.manifest_component_name())); - } - - fn package(&mut self, pkg: &PkgType, dst: &mut BTreeMap) { - if *pkg == PkgType::Rust { - // This is handled specially by `rust_package` later. - // Order is important, so don't call `rust_package` here. - return; - } - - let fallback = if pkg.use_docs_fallback() { DOCS_FALLBACK } else { &[] }; - let version_info = self.versions.version(&pkg).expect("failed to load package version"); - let mut is_present = version_info.present; - - // Never ship nightly-only components for other trains. - if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkg) { - is_present = false; // Pretend the component is entirely missing. - } - - macro_rules! tarball_name { - ($target_name:expr) => { - self.versions.tarball_name(pkg, $target_name).unwrap() - }; - } - let mut target_from_compressed_tar = |target_name| { - let target = Target::from_compressed_tar(self, &tarball_name!(target_name)); - if target.available { - return target; - } - for (substr, fallback_target) in fallback { - if target_name.contains(substr) { - let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target)); - // Fallbacks should typically be available on 'production' builds - // but may not be available for try builds, which only build one target by - // default. Ideally we'd gate this being a hard error on whether we're in a - // production build or not, but it's not information that's readily available - // here. - if !t.available { - eprintln!( - "{:?} not available for fallback", - tarball_name!(fallback_target) - ); - continue; - } - return t; - } - } - Target::unavailable() - }; - - let targets = pkg - .targets() - .iter() - .map(|name| { - let target = if is_present { - target_from_compressed_tar(name) - } else { - // If the component is not present for this build add it anyway but mark it as - // unavailable -- this way rustup won't allow upgrades without --force - Target::unavailable() - }; - (name.to_string(), target) - }) - .collect(); - - dst.insert(pkg.manifest_component_name(), Package { - version: version_info.version.unwrap_or_default(), - git_commit_hash: version_info.git_commit, - target: targets, - }); - } - - fn url(&self, path: &Path) -> String { - let file_name = path.file_name().unwrap().to_str().unwrap(); - format!("{}/{}/{}", self.s3_address, self.date, file_name) - } - - fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) { - self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml"); - self.write(&manifest.date, channel_name, "-date.txt"); - self.write( - manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(), - channel_name, - "-git-commit-hash.txt", - ); - } - - fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) { - let name = format!("channel-rust-{}{}", channel_name, suffix); - self.shipped_files.insert(name.clone()); - - let dst = self.output.join(name); - t!(fs::write(&dst, contents), format!("failed to create manifest {}", dst.display())); - } - - fn write_shipped_files(&self, path: &Path) { - let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::>(); - files.sort(); - let content = format!("{}\n", files.join("\n")); - - t!(std::fs::write(path, content.as_bytes())); - } -} diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 495cab582eb07..ba88e7e032175 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -13,12 +13,12 @@ const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; macro_rules! pkg_type { ( $($variant:ident = $component:literal $(; preview = true $(@$is_preview:tt)? )? ),+ $(,)? ) => { #[derive(Debug, Hash, Eq, PartialEq, Clone)] - pub(crate) enum PkgType { + pub enum PkgType { $($variant,)+ } impl PkgType { - pub(crate) fn is_preview(&self) -> bool { + pub fn is_preview(&self) -> bool { match self { $( $( $($is_preview)? PkgType::$variant => true, )? )+ _ => false, @@ -26,13 +26,13 @@ macro_rules! pkg_type { } /// First part of the tarball name. - pub(crate) fn tarball_component_name(&self) -> &str { + pub fn tarball_component_name(&self) -> &str { match self { $( PkgType::$variant => $component,)+ } } - pub(crate) fn all() -> &'static [PkgType] { + pub fn all() -> &'static [PkgType] { &[ $(PkgType::$variant),+ ] } } @@ -64,7 +64,7 @@ pkg_type! { impl PkgType { /// Component name in the manifest. In particular, this includes the `-preview` suffix where appropriate. - pub(crate) fn manifest_component_name(&self) -> String { + pub fn manifest_component_name(&self) -> String { if self.is_preview() { format!("{}-preview", self.tarball_component_name()) } else { @@ -150,14 +150,14 @@ pub(crate) struct VersionInfo { pub(crate) present: bool, } -pub(crate) struct Versions { +pub struct Versions { channel: String, dist_path: PathBuf, versions: HashMap, } impl Versions { - pub(crate) fn new(channel: &str, dist_path: &Path) -> Result { + pub fn new(channel: &str, dist_path: &Path) -> Result { Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() }) }