From 0917b2123fff6cf5357b5edd5db6d675313ab8bd Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 18 Sep 2020 12:44:25 +0200 Subject: [PATCH 1/5] build-manifest: move PkgType into the versions module --- src/tools/build-manifest/src/main.rs | 35 +++--------------------- src/tools/build-manifest/src/versions.rs | 27 ++++++++++++++++++ 2 files changed, 31 insertions(+), 31 deletions(-) create mode 100644 src/tools/build-manifest/src/versions.rs diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index ff9ee67763ba5..7a1878f749e70 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -4,8 +4,10 @@ //! via `x.py dist hash-and-sign`; the cmdline arguments are set up //! by rustbuild (in `src/bootstrap/dist.rs`). -use serde::Serialize; +mod versions; +use crate::versions::PkgType; +use serde::Serialize; use std::collections::BTreeMap; use std::collections::HashMap; use std::env; @@ -336,35 +338,6 @@ fn main() { .build(); } -enum PkgType { - RustSrc, - Cargo, - Rls, - RustAnalyzer, - Clippy, - Rustfmt, - LlvmTools, - Miri, - Other, -} - -impl PkgType { - fn from_component(component: &str) -> Self { - use PkgType::*; - match component { - "rust-src" => RustSrc, - "cargo" => Cargo, - "rls" | "rls-preview" => Rls, - "rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer, - "clippy" | "clippy-preview" => Clippy, - "rustfmt" | "rustfmt-preview" => Rustfmt, - "llvm-tools" | "llvm-tools-preview" => LlvmTools, - "miri" | "miri-preview" => Miri, - _ => Other, - } - } -} - impl Builder { fn build(&mut self) { self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); @@ -702,7 +675,7 @@ impl Builder { Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target), LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target), Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target), - Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target), + Other(_) => format!("{}-{}-{}.tar.gz", component, self.rust_release, target), } } diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs new file mode 100644 index 0000000000000..044c04914b187 --- /dev/null +++ b/src/tools/build-manifest/src/versions.rs @@ -0,0 +1,27 @@ +pub(crate) enum PkgType { + RustSrc, + Cargo, + Rls, + RustAnalyzer, + Clippy, + Rustfmt, + LlvmTools, + Miri, + Other(String), +} + +impl PkgType { + pub(crate) fn from_component(component: &str) -> Self { + match component { + "rust-src" => PkgType::RustSrc, + "cargo" => PkgType::Cargo, + "rls" | "rls-preview" => PkgType::Rls, + "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer, + "clippy" | "clippy-preview" => PkgType::Clippy, + "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt, + "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools, + "miri" | "miri-preview" => PkgType::Miri, + other => PkgType::Other(other.into()), + } + } +} From 3bddfea7e23341f788526f8b303b9a27038511e3 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 18 Sep 2020 14:40:01 +0200 Subject: [PATCH 2/5] build-manifest: stop receiving release numbers from bootstrap --- Cargo.lock | 1 + src/bootstrap/dist.rs | 10 +- src/tools/build-manifest/Cargo.toml | 1 + src/tools/build-manifest/README.md | 9 +- src/tools/build-manifest/src/main.rs | 77 +++++---------- src/tools/build-manifest/src/versions.rs | 114 +++++++++++++++++++++++ 6 files changed, 146 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3f777bc663dd..5a2dd23a7ae4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -230,6 +230,7 @@ dependencies = [ name = "build-manifest" version = "0.1.0" dependencies = [ + "anyhow", "serde", "serde_json", "toml", diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index f25ad50c9b774..991f8d4eea38b 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2356,15 +2356,9 @@ impl Step for HashSign { cmd.arg(sign); cmd.arg(distdir(builder)); cmd.arg(today.trim()); - cmd.arg(builder.rust_package_vers()); cmd.arg(addr); - cmd.arg(builder.package_vers(&builder.release_num("cargo"))); - cmd.arg(builder.package_vers(&builder.release_num("rls"))); - cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer"))); - cmd.arg(builder.package_vers(&builder.release_num("clippy"))); - cmd.arg(builder.package_vers(&builder.release_num("miri"))); - cmd.arg(builder.package_vers(&builder.release_num("rustfmt"))); - cmd.arg(builder.llvm_tools_package_vers()); + cmd.arg(&builder.config.channel); + cmd.arg(&builder.src); builder.create_dir(&distdir(builder)); diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index 0bbbabd29989e..2da224a54dce0 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -8,3 +8,4 @@ edition = "2018" toml = "0.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +anyhow = "1.0.32" diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index a80f36d496998..4d7d9f7da1874 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to ``` $ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \ - path/to/dist path/to/output 1970-01-01 \ - nightly nightly nightly nightly nightly nightly nightly nightly \ - http://example.com + path/to/dist path/to/output 1970-01-01 http://example.com \ + CHANNEL path/to/rust/repo ``` -In the future, if the tool complains about missing arguments just add more -`nightly`s in the middle. +Remember to replace `CHANNEL` with the channel you produced dist artifacts of +and `path/to/rust/repo` with the path to your checkout of the Rust repository. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 7a1878f749e70..c694948bac0ce 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -6,7 +6,7 @@ mod versions; -use crate::versions::PkgType; +use crate::versions::{PkgType, Versions}; use serde::Serialize; use std::collections::BTreeMap; use std::collections::HashMap; @@ -227,14 +227,7 @@ macro_rules! t { } struct Builder { - rust_release: String, - cargo_release: String, - rls_release: String, - rust_analyzer_release: String, - clippy_release: String, - rustfmt_release: String, - llvm_tools_release: String, - miri_release: String, + versions: Versions, input: PathBuf, output: PathBuf, @@ -281,15 +274,9 @@ fn main() { let input = PathBuf::from(args.next().unwrap()); let output = PathBuf::from(args.next().unwrap()); let date = args.next().unwrap(); - let rust_release = args.next().unwrap(); let s3_address = args.next().unwrap(); - let cargo_release = args.next().unwrap(); - let rls_release = args.next().unwrap(); - let rust_analyzer_release = args.next().unwrap(); - let clippy_release = args.next().unwrap(); - let miri_release = args.next().unwrap(); - let rustfmt_release = args.next().unwrap(); - let llvm_tools_release = args.next().unwrap(); + let channel = args.next().unwrap(); + let monorepo_path = args.next().unwrap(); // Do not ask for a passphrase while manually testing let mut passphrase = String::new(); @@ -299,14 +286,7 @@ fn main() { } Builder { - rust_release, - cargo_release, - rls_release, - rust_analyzer_release, - clippy_release, - rustfmt_release, - llvm_tools_release, - miri_release, + versions: Versions::new(&channel, Path::new(&monorepo_path)).unwrap(), input, output, @@ -363,10 +343,11 @@ impl Builder { self.check_toolstate(); self.digest_and_sign(); let manifest = self.build_manifest(); - self.write_channel_files(&self.rust_release, &manifest); - if self.rust_release != "beta" && self.rust_release != "nightly" { - self.write_channel_files("stable", &manifest); + let rust_version = self.versions.package_version(&PkgType::Rust).unwrap(); + self.write_channel_files(self.versions.channel(), &manifest); + if self.versions.channel() != rust_version { + self.write_channel_files(&rust_version, &manifest); } } @@ -473,7 +454,7 @@ impl Builder { // The compiler libraries are not stable for end users, and they're also huge, so we only // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible // for users to install the additional component manually, if needed. - if self.rust_release == "nightly" { + if self.versions.channel() == "nightly" { self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]); } @@ -511,7 +492,7 @@ impl Builder { } fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option { - let filename = self.filename("rust", host); + let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap(); let digest = self.digests.remove(&filename)?; let xz_filename = filename.replace(".tar.gz", ".tar.xz"); let xz_digest = self.digests.remove(&xz_filename); @@ -610,7 +591,7 @@ impl Builder { .unwrap_or_default(); // `is_present` defaults to `false` here. // Never ship nightly-only components for other trains. - if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { + if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { is_present = false; // Pretend the component is entirely missing. } @@ -619,7 +600,10 @@ impl Builder { .map(|name| { if is_present { // The component generally exists, but it might still be missing for this target. - let filename = self.filename(pkgname, name); + let filename = self + .versions + .tarball_name(&PkgType::from_component(pkgname), name) + .unwrap(); let digest = match self.digests.remove(&filename) { Some(digest) => digest, // This component does not exist for this target -- skip it. @@ -662,23 +646,6 @@ impl Builder { format!("{}/{}/{}", self.s3_address, self.date, filename) } - fn filename(&self, component: &str, target: &str) -> String { - use PkgType::*; - match PkgType::from_component(component) { - RustSrc => format!("rust-src-{}.tar.gz", self.rust_release), - Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target), - Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target), - RustAnalyzer => { - format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target) - } - Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target), - Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target), - LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target), - Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target), - Other(_) => format!("{}-{}-{}.tar.gz", component, self.rust_release, target), - } - } - fn cached_version(&self, component: &str) -> &Option { use PkgType::*; match PkgType::from_component(component) { @@ -707,20 +674,24 @@ impl Builder { } } - fn version(&self, component: &str, target: &str) -> Option { + fn version(&mut self, component: &str, target: &str) -> Option { self.untar(component, target, |filename| format!("{}/version", filename)) } - fn git_commit_hash(&self, component: &str, target: &str) -> Option { + fn git_commit_hash(&mut self, component: &str, target: &str) -> Option { self.untar(component, target, |filename| format!("{}/git-commit-hash", filename)) } - fn untar(&self, component: &str, target: &str, dir: F) -> Option + fn untar(&mut self, component: &str, target: &str, dir: F) -> Option where F: FnOnce(String) -> String, { + let filename = self + .versions + .tarball_name(&PkgType::from_component(component), target) + .expect("failed to retrieve the tarball path"); + let mut cmd = Command::new("tar"); - let filename = self.filename(component, target); cmd.arg("xf") .arg(self.input.join(&filename)) .arg(dir(filename.replace(".tar.gz", ""))) diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 044c04914b187..151cfa0d9c6b1 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -1,4 +1,10 @@ +use anyhow::{Context, Error}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Hash, Eq, PartialEq, Clone)] pub(crate) enum PkgType { + Rust, RustSrc, Cargo, Rls, @@ -13,6 +19,7 @@ pub(crate) enum PkgType { impl PkgType { pub(crate) fn from_component(component: &str) -> Self { match component { + "rust" => PkgType::Rust, "rust-src" => PkgType::RustSrc, "cargo" => PkgType::Cargo, "rls" | "rls-preview" => PkgType::Rls, @@ -24,4 +31,111 @@ impl PkgType { other => PkgType::Other(other.into()), } } + + fn rust_monorepo_path(&self) -> Option<&'static str> { + match self { + PkgType::Cargo => Some("src/tools/cargo"), + PkgType::Rls => Some("src/tools/rls"), + PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"), + PkgType::Clippy => Some("src/tools/clippy"), + PkgType::Rustfmt => Some("src/tools/rustfmt"), + PkgType::Miri => Some("src/tools/miri"), + PkgType::Rust => None, + PkgType::RustSrc => None, + PkgType::LlvmTools => None, + PkgType::Other(_) => None, + } + } + + fn tarball_component_name(&self) -> &str { + match self { + PkgType::Rust => "rust", + PkgType::RustSrc => "rust-src", + PkgType::Cargo => "cargo", + PkgType::Rls => "rls", + PkgType::RustAnalyzer => "rust-analyzer", + PkgType::Clippy => "clippy", + PkgType::Rustfmt => "rustfmt", + PkgType::LlvmTools => "llvm-tools", + PkgType::Miri => "miri", + PkgType::Other(component) => component, + } + } +} + +pub(crate) struct Versions { + channel: String, + rustc_version: String, + monorepo_root: PathBuf, + package_versions: HashMap, +} + +impl Versions { + pub(crate) fn new(channel: &str, monorepo_root: &Path) -> Result { + Ok(Self { + channel: channel.into(), + rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version")) + .context("failed to read the rustc version from src/version")? + .trim() + .to_string(), + monorepo_root: monorepo_root.into(), + package_versions: HashMap::new(), + }) + } + + pub(crate) fn channel(&self) -> &str { + &self.channel + } + + pub(crate) fn tarball_name( + &mut self, + package: &PkgType, + target: &str, + ) -> Result { + Ok(format!( + "{}-{}-{}.tar.gz", + package.tarball_component_name(), + self.package_version(package).with_context(|| format!( + "failed to get the package version for component {:?}", + package, + ))?, + target + )) + } + + pub(crate) fn package_version(&mut self, package: &PkgType) -> Result { + match self.package_versions.get(package) { + Some(release) => Ok(release.clone()), + None => { + let version = match package.rust_monorepo_path() { + Some(path) => { + let path = self.monorepo_root.join(path).join("Cargo.toml"); + let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?; + cargo_toml.package.version + } + None => self.rustc_version.clone(), + }; + + let release = match self.channel.as_str() { + "stable" => version, + "beta" => "beta".into(), + "nightly" => "nightly".into(), + _ => format!("{}-dev", version), + }; + + self.package_versions.insert(package.clone(), release.clone()); + Ok(release) + } + } + } +} + +#[derive(serde::Deserialize)] +struct CargoToml { + package: CargoTomlPackage, +} + +#[derive(serde::Deserialize)] +struct CargoTomlPackage { + version: String, } From 89ffab76b7a17355f8fa0c82b06e6fb7ec48f09c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 24 Sep 2020 14:13:40 +0200 Subject: [PATCH 3/5] build-manifest: refactor detecting package versions --- Cargo.lock | 2 + src/tools/build-manifest/Cargo.toml | 2 + src/tools/build-manifest/src/main.rs | 143 ++--------------------- src/tools/build-manifest/src/versions.rs | 103 +++++++++++++++- 4 files changed, 118 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5a2dd23a7ae4f..f18db340aa638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,8 +231,10 @@ name = "build-manifest" version = "0.1.0" dependencies = [ "anyhow", + "flate2", "serde", "serde_json", + "tar", "toml", ] diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index 2da224a54dce0..4f89c31936dda 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -9,3 +9,5 @@ toml = "0.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" anyhow = "1.0.32" +flate2 = "1.0.16" +tar = "0.4.29" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index c694948bac0ce..d8813f95be46f 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -236,24 +236,6 @@ struct Builder { s3_address: String, date: String, - rust_version: Option, - cargo_version: Option, - rls_version: Option, - rust_analyzer_version: Option, - clippy_version: Option, - rustfmt_version: Option, - llvm_tools_version: Option, - miri_version: Option, - - rust_git_commit_hash: Option, - cargo_git_commit_hash: Option, - rls_git_commit_hash: Option, - rust_analyzer_git_commit_hash: Option, - clippy_git_commit_hash: Option, - rustfmt_git_commit_hash: Option, - llvm_tools_git_commit_hash: Option, - miri_git_commit_hash: Option, - should_sign: bool, } @@ -286,7 +268,7 @@ fn main() { } Builder { - versions: Versions::new(&channel, Path::new(&monorepo_path)).unwrap(), + versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(), input, output, @@ -295,24 +277,6 @@ fn main() { s3_address, date, - rust_version: None, - cargo_version: None, - rls_version: None, - rust_analyzer_version: None, - clippy_version: None, - rustfmt_version: None, - llvm_tools_version: None, - miri_version: None, - - rust_git_commit_hash: None, - cargo_git_commit_hash: None, - rls_git_commit_hash: None, - rust_analyzer_git_commit_hash: None, - clippy_git_commit_hash: None, - rustfmt_git_commit_hash: None, - llvm_tools_git_commit_hash: None, - miri_git_commit_hash: None, - should_sign, } .build(); @@ -320,26 +284,6 @@ fn main() { impl Builder { fn build(&mut self) { - self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); - self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu"); - self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu"); - self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu"); - self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu"); - self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu"); - self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu"); - self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu"); - - self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu"); - self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu"); - self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu"); - self.rust_analyzer_git_commit_hash = - self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu"); - self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu"); - self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu"); - self.llvm_tools_git_commit_hash = - self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu"); - self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu"); - self.check_toolstate(); self.digest_and_sign(); let manifest = self.build_manifest(); @@ -368,8 +312,7 @@ impl Builder { // Mark some tools as missing based on toolstate. if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") { println!("Miri tests are not passing, removing component"); - self.miri_version = None; - self.miri_git_commit_hash = None; + self.versions.disable_version(&PkgType::Miri); } } @@ -471,13 +414,10 @@ impl Builder { } fn rust_package(&mut self, manifest: &Manifest) -> Package { + let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball"); let mut pkg = Package { - version: self - .cached_version("rust") - .as_ref() - .expect("Couldn't find Rust version") - .clone(), - git_commit_hash: self.cached_git_commit_hash("rust").clone(), + version: version_info.version.expect("missing Rust version"), + git_commit_hash: version_info.git_commit, target: BTreeMap::new(), }; for host in HOSTS { @@ -583,12 +523,11 @@ impl Builder { } fn package(&mut self, pkgname: &str, dst: &mut BTreeMap, targets: &[&str]) { - let (version, mut is_present) = self - .cached_version(pkgname) - .as_ref() - .cloned() - .map(|version| (version, true)) - .unwrap_or_default(); // `is_present` defaults to `false` here. + let version_info = self + .versions + .version(&PkgType::from_component(pkgname)) + .expect("failed to load package version"); + let mut is_present = version_info.present; // Never ship nightly-only components for other trains. if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { @@ -635,8 +574,8 @@ impl Builder { dst.insert( pkgname.to_string(), Package { - version, - git_commit_hash: self.cached_git_commit_hash(pkgname).clone(), + version: version_info.version.unwrap_or_default(), + git_commit_hash: version_info.git_commit, target: targets, }, ); @@ -646,64 +585,6 @@ impl Builder { format!("{}/{}/{}", self.s3_address, self.date, filename) } - fn cached_version(&self, component: &str) -> &Option { - use PkgType::*; - match PkgType::from_component(component) { - Cargo => &self.cargo_version, - Rls => &self.rls_version, - RustAnalyzer => &self.rust_analyzer_version, - Clippy => &self.clippy_version, - Rustfmt => &self.rustfmt_version, - LlvmTools => &self.llvm_tools_version, - Miri => &self.miri_version, - _ => &self.rust_version, - } - } - - fn cached_git_commit_hash(&self, component: &str) -> &Option { - use PkgType::*; - match PkgType::from_component(component) { - Cargo => &self.cargo_git_commit_hash, - Rls => &self.rls_git_commit_hash, - RustAnalyzer => &self.rust_analyzer_git_commit_hash, - Clippy => &self.clippy_git_commit_hash, - Rustfmt => &self.rustfmt_git_commit_hash, - LlvmTools => &self.llvm_tools_git_commit_hash, - Miri => &self.miri_git_commit_hash, - _ => &self.rust_git_commit_hash, - } - } - - fn version(&mut self, component: &str, target: &str) -> Option { - self.untar(component, target, |filename| format!("{}/version", filename)) - } - - fn git_commit_hash(&mut self, component: &str, target: &str) -> Option { - self.untar(component, target, |filename| format!("{}/git-commit-hash", filename)) - } - - fn untar(&mut self, component: &str, target: &str, dir: F) -> Option - where - F: FnOnce(String) -> String, - { - let filename = self - .versions - .tarball_name(&PkgType::from_component(component), target) - .expect("failed to retrieve the tarball path"); - - let mut cmd = Command::new("tar"); - cmd.arg("xf") - .arg(self.input.join(&filename)) - .arg(dir(filename.replace(".tar.gz", ""))) - .arg("-O"); - let output = t!(cmd.output()); - if output.status.success() { - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) - } else { - None - } - } - fn hash(&self, path: &Path) -> String { let sha = t!(Command::new("shasum") .arg("-a") diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 151cfa0d9c6b1..3b5caf1cd7b91 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -1,6 +1,12 @@ use anyhow::{Context, Error}; +use flate2::read::GzDecoder; use std::collections::HashMap; +use std::fs::File; +use std::io::Read; use std::path::{Path, PathBuf}; +use tar::Archive; + +const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; #[derive(Debug, Hash, Eq, PartialEq, Clone)] pub(crate) enum PkgType { @@ -61,17 +67,46 @@ impl PkgType { PkgType::Other(component) => component, } } + + fn should_use_rust_version(&self) -> bool { + match self { + PkgType::Cargo => false, + PkgType::Rls => false, + PkgType::RustAnalyzer => false, + PkgType::Clippy => false, + PkgType::Rustfmt => false, + PkgType::LlvmTools => false, + PkgType::Miri => false, + + PkgType::Rust => true, + PkgType::RustSrc => true, + PkgType::Other(_) => true, + } + } +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct VersionInfo { + pub(crate) version: Option, + pub(crate) git_commit: Option, + pub(crate) present: bool, } pub(crate) struct Versions { channel: String, rustc_version: String, monorepo_root: PathBuf, + dist_path: PathBuf, package_versions: HashMap, + versions: HashMap, } impl Versions { - pub(crate) fn new(channel: &str, monorepo_root: &Path) -> Result { + pub(crate) fn new( + channel: &str, + dist_path: &Path, + monorepo_root: &Path, + ) -> Result { Ok(Self { channel: channel.into(), rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version")) @@ -79,7 +114,9 @@ impl Versions { .trim() .to_string(), monorepo_root: monorepo_root.into(), + dist_path: dist_path.into(), package_versions: HashMap::new(), + versions: HashMap::new(), }) } @@ -87,6 +124,70 @@ impl Versions { &self.channel } + pub(crate) fn version(&mut self, mut package: &PkgType) -> Result { + if package.should_use_rust_version() { + package = &PkgType::Rust; + } + + match self.versions.get(package) { + Some(version) => Ok(version.clone()), + None => { + let version_info = self.load_version_from_tarball(package)?; + self.versions.insert(package.clone(), version_info.clone()); + Ok(version_info) + } + } + } + + fn load_version_from_tarball(&mut self, package: &PkgType) -> Result { + let tarball_name = self.tarball_name(package, DEFAULT_TARGET)?; + let tarball = self.dist_path.join(tarball_name); + + let file = match File::open(&tarball) { + Ok(file) => file, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // Missing tarballs do not return an error, but return empty data. + return Ok(VersionInfo::default()); + } + Err(err) => return Err(err.into()), + }; + let mut tar = Archive::new(GzDecoder::new(file)); + + let mut version = None; + let mut git_commit = None; + for entry in tar.entries()? { + let mut entry = entry?; + + let dest; + match entry.path()?.components().nth(1).and_then(|c| c.as_os_str().to_str()) { + Some("version") => dest = &mut version, + Some("git-commit-hash") => dest = &mut git_commit, + _ => continue, + } + let mut buf = String::new(); + entry.read_to_string(&mut buf)?; + *dest = Some(buf); + + // Short circuit to avoid reading the whole tar file if not necessary. + if version.is_some() && git_commit.is_some() { + break; + } + } + + Ok(VersionInfo { version, git_commit, present: true }) + } + + pub(crate) fn disable_version(&mut self, package: &PkgType) { + match self.versions.get_mut(package) { + Some(version) => { + *version = VersionInfo::default(); + } + None => { + self.versions.insert(package.clone(), VersionInfo::default()); + } + } + } + pub(crate) fn tarball_name( &mut self, package: &PkgType, From 73d9c24b3edf0f87a6e1c78ca56349f94ced5788 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 24 Sep 2020 15:23:35 +0200 Subject: [PATCH 4/5] build-manifest: add documentation on the PkgType methods --- src/tools/build-manifest/src/versions.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 3b5caf1cd7b91..9f37b22c4941d 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -38,6 +38,8 @@ impl PkgType { } } + /// The directory containing the `Cargo.toml` of this component inside the monorepo, to + /// retrieve the source code version. If `None` is returned Rust's version will be used. fn rust_monorepo_path(&self) -> Option<&'static str> { match self { PkgType::Cargo => Some("src/tools/cargo"), @@ -53,6 +55,7 @@ impl PkgType { } } + /// First part of the tarball name. fn tarball_component_name(&self) -> &str { match self { PkgType::Rust => "rust", @@ -68,6 +71,8 @@ impl PkgType { } } + /// Whether this package has the same version as Rust itself, or has its own `version` and + /// `git-commit-hash` files inside the tarball. fn should_use_rust_version(&self) -> bool { match self { PkgType::Cargo => false, From 0749ad02d0b8429e2a3a50158994268c0b0af044 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 25 Sep 2020 15:36:16 +0200 Subject: [PATCH 5/5] build-manifest: handle rust-src being target-independent --- src/tools/build-manifest/src/versions.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 9f37b22c4941d..d949dff72798b 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -88,6 +88,11 @@ impl PkgType { PkgType::Other(_) => true, } } + + /// Whether this package is target-independent or not. + fn target_independent(&self) -> bool { + *self == PkgType::RustSrc + } } #[derive(Debug, Default, Clone)] @@ -198,15 +203,15 @@ impl Versions { package: &PkgType, target: &str, ) -> Result { - Ok(format!( - "{}-{}-{}.tar.gz", - package.tarball_component_name(), - self.package_version(package).with_context(|| format!( - "failed to get the package version for component {:?}", - package, - ))?, - target - )) + let component_name = package.tarball_component_name(); + let version = self.package_version(package).with_context(|| { + format!("failed to get the package version for component {:?}", package,) + })?; + if package.target_independent() { + Ok(format!("{}-{}.tar.gz", component_name, version)) + } else { + Ok(format!("{}-{}-{}.tar.gz", component_name, version, target)) + } } pub(crate) fn package_version(&mut self, package: &PkgType) -> Result {