From d9116132809034c671f17143b8ad7948bf714798 Mon Sep 17 00:00:00 2001 From: Jeffery To Date: Fri, 13 Oct 2023 23:05:29 +0800 Subject: [PATCH 1/3] Allow bootstrap cache path to be set by environment variable This allows the bootstrap cache path to be set by the `RUSTC_BOOTSTRAP_CACHE` environment variable. Setting the bootstrap cache path to an external location can help to speed up builds in cases where the build directory is not kept between builds, e.g. in CI or other automated build systems. --- src/bootstrap/bootstrap.py | 2 +- src/bootstrap/src/core/download.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 0d604c0d3e5e0..04146eb78d713 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -557,7 +557,7 @@ def download_toolchain(self): shutil.rmtree(bin_root) key = self.stage0_compiler.date - cache_dst = os.path.join(self.build_dir, "cache") + cache_dst = os.getenv('RUSTC_BOOTSTRAP_CACHE', os.path.join(self.build_dir, "cache")) rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): os.makedirs(rustc_cache) diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 185089a646bfa..db83d8a214f1d 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -578,7 +578,11 @@ impl Config { return; } - let cache_dst = self.out.join("cache"); + let cache_dst = match env::var_os("RUSTC_BOOTSTRAP_CACHE") { + Some(v) => PathBuf::from(v), + None => self.out.join("cache"), + }; + let cache_dir = cache_dst.join(key); if !cache_dir.exists() { t!(fs::create_dir_all(&cache_dir)); @@ -705,7 +709,10 @@ download-rustc = false let llvm_assertions = self.llvm_assertions; let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); - let cache_dst = self.out.join("cache"); + let cache_dst = match env::var_os("RUSTC_BOOTSTRAP_CACHE") { + Some(v) => PathBuf::from(v), + None => self.out.join("cache"), + }; let rustc_cache = cache_dst.join(cache_prefix); if !rustc_cache.exists() { t!(fs::create_dir_all(&rustc_cache)); From c98e25bab9503741bcafd20f79a230348d6b7884 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Mon, 4 Mar 2024 11:06:24 +0100 Subject: [PATCH 2/3] Add a build option to specify the bootstrap cache Setting the bootstrap cache path to an external location can help to speed up builds in cases where the build directory is not kept between builds, e.g. in CI or other automated build systems. --- config.example.toml | 4 ++++ src/bootstrap/bootstrap.py | 4 +++- src/bootstrap/configure.py | 1 + src/bootstrap/src/core/config/config.rs | 4 ++++ src/bootstrap/src/core/download.rs | 13 +++++-------- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/config.example.toml b/config.example.toml index cef33a7905a3b..ce4d80a945290 100644 --- a/config.example.toml +++ b/config.example.toml @@ -300,6 +300,10 @@ # This is only useful for verifying that rustc generates reproducible builds. #full-bootstrap = false +# Set the bootstrap/download cache path. It is useful when building rust +# repeatedly in a CI invironment. +# bootstrap-cache-path = /shared/cache + # Enable a build of the extended Rust tool set which is not only the compiler # but also tools such as Cargo. This will also produce "combined installers" # which are used to install Rust and Cargo together. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 04146eb78d713..6e49bcc974469 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -557,7 +557,9 @@ def download_toolchain(self): shutil.rmtree(bin_root) key = self.stage0_compiler.date - cache_dst = os.getenv('RUSTC_BOOTSTRAP_CACHE', os.path.join(self.build_dir, "cache")) + cache_dst = (self.get_toml('bootstrap-cache-path', 'build') or + os.path.join(self.build_dir, "cache")) + rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): os.makedirs(rustc_cache) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 8b65e8ff9c3e1..4257c0f7991a6 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -149,6 +149,7 @@ def v(*args): # (others are conditionally saved). o("manage-submodules", "build.submodules", "let the build manage the git submodules") o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)") +o("bootstrap-cache-path", "build.bootstrap-cache-path", "use provided path for the bootstrap cache") o("extended", "build.extended", "build an extended rust tool set") v("tools", None, "List of extended tools will be installed") diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 875a4efae02fc..326f8f57173a1 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -161,6 +161,7 @@ pub struct Config { pub vendor: bool, pub target_config: HashMap, pub full_bootstrap: bool, + pub bootstrap_cache_path: Option, pub extended: bool, pub tools: Option>, pub sanitizers: bool, @@ -827,6 +828,7 @@ define_config! { locked_deps: Option = "locked-deps", vendor: Option = "vendor", full_bootstrap: Option = "full-bootstrap", + bootstrap_cache_path: Option = "bootstrap-cache-path", extended: Option = "extended", tools: Option> = "tools", verbose: Option = "verbose", @@ -1389,6 +1391,7 @@ impl Config { locked_deps, vendor, full_bootstrap, + bootstrap_cache_path, extended, tools, verbose, @@ -1477,6 +1480,7 @@ impl Config { config.reuse = reuse.map(PathBuf::from); config.submodules = submodules; config.android_ndk = android_ndk; + config.bootstrap_cache_path = bootstrap_cache_path; set(&mut config.low_priority, low_priority); set(&mut config.compiler_docs, compiler_docs); set(&mut config.library_docs_private_items, library_docs_private_items); diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index db83d8a214f1d..27829eab9379d 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -578,10 +578,8 @@ impl Config { return; } - let cache_dst = match env::var_os("RUSTC_BOOTSTRAP_CACHE") { - Some(v) => PathBuf::from(v), - None => self.out.join("cache"), - }; + let cache_dst = + self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); let cache_dir = cache_dst.join(key); if !cache_dir.exists() { @@ -709,10 +707,9 @@ download-rustc = false let llvm_assertions = self.llvm_assertions; let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); - let cache_dst = match env::var_os("RUSTC_BOOTSTRAP_CACHE") { - Some(v) => PathBuf::from(v), - None => self.out.join("cache"), - }; + let cache_dst = + self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); + let rustc_cache = cache_dst.join(cache_prefix); if !rustc_cache.exists() { t!(fs::create_dir_all(&rustc_cache)); diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 9a50ad4437e73..5ff367ea3ad9f 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -136,4 +136,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "`x install` now skips providing tarball sources (under 'build/dist' path) to speed up the installation process.", }, + ChangeInfo { + change_id: 121976, + severity: ChangeSeverity::Info, + summary: "A new `boostrap-cache-path` option has been introduced. Set it in your config.toml to use a different path for the download cache.", + }, ]; From 0a80f9a4880ad2b6d94c660a6d94288c9a8490d5 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Wed, 6 Mar 2024 18:12:35 +0100 Subject: [PATCH 3/3] Update src/bootstrap/src/utils/change_tracker.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Onur Özkan --- src/bootstrap/src/utils/change_tracker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 5ff367ea3ad9f..d166b84e51fc5 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -139,6 +139,6 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ ChangeInfo { change_id: 121976, severity: ChangeSeverity::Info, - summary: "A new `boostrap-cache-path` option has been introduced. Set it in your config.toml to use a different path for the download cache.", + summary: "A new `boostrap-cache-path` option has been introduced which can be utilized to modify the cache path for bootstrap.", }, ];