Skip to content

Commit

Permalink
Auto merge of rust-lang#115795 - Kobzol:opt-dist-custom, r=Mark-Simul…
Browse files Browse the repository at this point in the history
…acrum

Refactor `opt-dist` to simplify local building

This PR refactors the `opt-dist` tool to make it easier to invoke it locally, outside of CI, and thus simplify building PGO/BOLT optimized `rustc` builds e.g. for distro maintainers. It should also make it easier to run the PGO/BOLT workflow locally e.g. to profile performance or debug issues (looking at you, rust-lang#115554).
  • Loading branch information
bors committed Sep 18, 2023
2 parents de68911 + ee451f8 commit cbcf9a5
Show file tree
Hide file tree
Showing 15 changed files with 423 additions and 277 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ jobs:
- name: dist-x86_64-msvc
env:
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
os: windows-2019-8core-32gb
- name: dist-i686-msvc
Expand Down
76 changes: 72 additions & 4 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -858,14 +858,38 @@ dependencies = [
"winapi",
]

[[package]]
name = "darling"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
dependencies = [
"darling_core 0.14.4",
"darling_macro 0.14.4",
]

[[package]]
name = "darling"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
dependencies = [
"darling_core",
"darling_macro",
"darling_core 0.20.3",
"darling_macro 0.20.3",
]

[[package]]
name = "darling_core"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn 1.0.109",
]

[[package]]
Expand All @@ -882,13 +906,24 @@ dependencies = [
"syn 2.0.29",
]

[[package]]
name = "darling_macro"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
dependencies = [
"darling_core 0.14.4",
"quote",
"syn 1.0.109",
]

[[package]]
name = "darling_macro"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core",
"darling_core 0.20.3",
"quote",
"syn 2.0.29",
]
Expand Down Expand Up @@ -919,6 +954,37 @@ dependencies = [
"syn 1.0.109",
]

[[package]]
name = "derive_builder"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
dependencies = [
"derive_builder_macro",
]

[[package]]
name = "derive_builder_core"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
dependencies = [
"darling 0.14.4",
"proc-macro2",
"quote",
"syn 1.0.109",
]

[[package]]
name = "derive_builder_macro"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
dependencies = [
"derive_builder_core",
"syn 1.0.109",
]

[[package]]
name = "derive_more"
version = "0.99.17"
Expand All @@ -938,7 +1004,7 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
dependencies = [
"darling",
"darling 0.20.3",
"proc-macro2",
"quote",
"syn 2.0.29",
Expand Down Expand Up @@ -2584,6 +2650,8 @@ dependencies = [
"anyhow",
"build_helper",
"camino",
"clap",
"derive_builder",
"env_logger 0.10.0",
"fs_extra",
"glob",
Expand Down
2 changes: 1 addition & 1 deletion src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
--set rust.lto=thin

ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
--host $HOSTS --target $HOSTS \
--include-default-paths \
build-manifest bootstrap
Expand Down
2 changes: 1 addition & 1 deletion src/ci/github-actions/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,7 @@ jobs:
--target=x86_64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
<<: *job-windows-8c

Expand Down
2 changes: 2 additions & 0 deletions src/tools/opt-dist/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
glob = "0.3"
tempfile = "3.5"
derive_builder = "0.12"
clap = { version = "4", features = ["derive"] }
108 changes: 108 additions & 0 deletions src/tools/opt-dist/src/environment.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
use camino::Utf8PathBuf;
use derive_builder::Builder;

#[derive(Builder)]
pub struct Environment {
host_triple: String,
python_binary: String,
/// The rustc checkout, where the compiler source is located.
checkout_dir: Utf8PathBuf,
/// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
/// this directory before `opt-dist` is started.
build_dir: Utf8PathBuf,
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
/// will be stored.
artifact_dir: Utf8PathBuf,
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
host_llvm_dir: Utf8PathBuf,
/// List of test paths that should be skipped when testing the optimized artifacts.
skipped_tests: Vec<String>,
/// Directory containing a pre-built rustc-perf checkout.
#[builder(default)]
prebuilt_rustc_perf: Option<Utf8PathBuf>,
use_bolt: bool,
shared_llvm: bool,
}

impl Environment {
pub fn host_triple(&self) -> &str {
&self.host_triple
}

pub fn python_binary(&self) -> &str {
&self.python_binary
}

pub fn checkout_path(&self) -> Utf8PathBuf {
self.checkout_dir.clone()
}

pub fn build_root(&self) -> Utf8PathBuf {
self.build_dir.clone()
}

pub fn build_artifacts(&self) -> Utf8PathBuf {
self.build_root().join("build").join(&self.host_triple)
}

pub fn artifact_dir(&self) -> Utf8PathBuf {
self.artifact_dir.clone()
}

pub fn cargo_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("cargo{}", executable_extension()))
}

pub fn rustc_stage_0(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage0")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}

pub fn rustc_stage_2(&self) -> Utf8PathBuf {
self.build_artifacts()
.join("stage2")
.join("bin")
.join(format!("rustc{}", executable_extension()))
}

pub fn prebuilt_rustc_perf(&self) -> Option<Utf8PathBuf> {
self.prebuilt_rustc_perf.clone()
}

/// Path to the built rustc-perf benchmark suite.
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
self.artifact_dir.join("rustc-perf")
}

pub fn host_llvm_dir(&self) -> Utf8PathBuf {
self.host_llvm_dir.clone()
}

pub fn use_bolt(&self) -> bool {
self.use_bolt
}

pub fn supports_shared_llvm(&self) -> bool {
self.shared_llvm
}

pub fn skipped_tests(&self) -> &[String] {
&self.skipped_tests
}
}

/// What is the extension of binary executables on this platform?
#[cfg(target_family = "unix")]
pub fn executable_extension() -> &'static str {
""
}

#[cfg(target_family = "windows")]
pub fn executable_extension() -> &'static str {
".exe"
}
58 changes: 0 additions & 58 deletions src/tools/opt-dist/src/environment/linux.rs

This file was deleted.

Loading

0 comments on commit cbcf9a5

Please sign in to comment.