Skip to content

Commit

Permalink
Auto merge of #130040 - onur-ozkan:llvm-tools-with-ci-rustc, r=Kobzol
Browse files Browse the repository at this point in the history 
unify `llvm-bitcode-linker`, `wasm-component-ld` and llvm-tools logics

To use the precompiled `ci-rustc` in CI, we need to install `llvm-bitcode-linker` and LLVM tools into ci-rustc's sysroot. Without them some CI pipelines may fail, as shown [here](#122709 (comment)).

Blocker for #122709
  • Loading branch information
@bors
bors committed Sep 12, 2024
2 parents 2e8db5e + 4967093 commit adaff53
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 50 deletions.
64 changes: 19 additions & 45 deletions src/bootstrap/src/core/build_steps/compile.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use crate::utils::exec::command;
use crate::utils::helpers::{
self, exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, symlink_dir, t, up_to_date,
};
use crate::{CLang, Compiler, DependencyType, GitRepo, Mode, LLVM_TOOLS};
use crate::{CLang, Compiler, DependencyType, GitRepo, Mode};

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Std {
Expand Down Expand Up @@ -1908,52 +1908,26 @@ impl Step for Assemble {
// delegates to the `rust-lld` binary for linking and then runs
// logic to create the final binary. This is used by the
// `wasm32-wasip2` target of Rust.
if builder.tool_enabled("wasm-component-ld") {
let wasm_component_ld_exe =
builder.ensure(crate::core::build_steps::tool::WasmComponentLd {
compiler: build_compiler,
target: target_compiler.host,
});
builder.copy_link(
&wasm_component_ld_exe,
&libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()),
);
}
dist::maybe_install_wasm_component_ld(
builder,
build_compiler,
target_compiler.host,
&libdir_bin,
false,
);

if builder.config.llvm_enabled(target_compiler.host) {
let llvm::LlvmResult { llvm_config, .. } =
builder.ensure(llvm::Llvm { target: target_compiler.host });
if !builder.config.dry_run() && builder.config.llvm_tools_enabled {
let llvm_bin_dir =
command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
let llvm_bin_dir = Path::new(llvm_bin_dir.trim());

// Since we've already built the LLVM tools, install them to the sysroot.
// This is the equivalent of installing the `llvm-tools-preview` component via
// rustup, and lets developers use a locally built toolchain to
// build projects that expect llvm tools to be present in the sysroot
// (e.g. the `bootimage` crate).
for tool in LLVM_TOOLS {
let tool_exe = exe(tool, target_compiler.host);
let src_path = llvm_bin_dir.join(&tool_exe);
// When using `download-ci-llvm`, some of the tools
// may not exist, so skip trying to copy them.
if src_path.exists() {
builder.copy_link(&src_path, &libdir_bin.join(&tool_exe));
}
}
}
}
dist::maybe_install_llvm_tools(builder, target_compiler.host, &libdir_bin, false);

if builder.config.llvm_bitcode_linker_enabled {
let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
compiler: build_compiler,
target: target_compiler.host,
extra_features: vec![],
});
let tool_exe = exe("llvm-bitcode-linker", target_compiler.host);
builder.copy_link(&src_path, &libdir_bin.join(tool_exe));
}
let self_contained_bin_dir = libdir_bin.join("self-contained");
t!(fs::create_dir_all(&self_contained_bin_dir));

dist::maybe_install_llvm_bitcode_linker(
builder,
build_compiler,
target_compiler.host,
&self_contained_bin_dir,
false,
);

// Ensure that `libLLVM.so` ends up in the newly build compiler directory,
// so that it can be found when the newly built `rustc` is run.
Expand Down
107 changes: 102 additions & 5 deletions src/bootstrap/src/core/build_steps/dist.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,11 +473,29 @@ impl Step for Rustc {
);
}
}
if builder.tool_enabled("wasm-component-ld") {
let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin");
let ld = exe("wasm-component-ld", compiler.host);
builder.copy_link(&src_dir.join(&ld), &dst_dir.join(&ld));
}

let builder_compiler =
builder.compiler_for(builder.top_stage, builder.config.build, compiler.host);

maybe_install_wasm_component_ld(
builder,
builder_compiler,
compiler.host,
&dst_dir,
true,
);

let self_contained_bin_dir = dst_dir.join("self-contained");
t!(fs::create_dir_all(&self_contained_bin_dir));
maybe_install_llvm_bitcode_linker(
builder,
builder_compiler,
compiler.host,
&self_contained_bin_dir,
true,
);

maybe_install_llvm_tools(builder, compiler.host, &dst_dir, true);

// Man pages
t!(fs::create_dir_all(image.join("share/man/man1")));
Expand Down Expand Up @@ -2086,6 +2104,85 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection
}
}

/// Maybe add LLVM tools to the rustc sysroot.
pub fn maybe_install_llvm_tools(
builder: &Builder<'_>,
target: TargetSelection,
dst_dir: &Path,
dereference_symlinks: bool,
) {
if builder.config.llvm_enabled(target) {
let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target });
if !builder.config.dry_run() && builder.config.llvm_tools_enabled {
let llvm_bin_dir =
command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
let llvm_bin_dir = Path::new(llvm_bin_dir.trim());

// Since we've already built the LLVM tools, install them to the sysroot.
// This is the equivalent of installing the `llvm-tools-preview` component via
// rustup, and lets developers use a locally built toolchain to
// build projects that expect llvm tools to be present in the sysroot
// (e.g. the `bootimage` crate).
for tool in LLVM_TOOLS {
let tool_exe = exe(tool, target);
let src_path = llvm_bin_dir.join(&tool_exe);
// When using `download-ci-llvm`, some of the tools
// may not exist, so skip trying to copy them.
if src_path.exists() {
builder.copy_link_internal(
&src_path,
&dst_dir.join(&tool_exe),
dereference_symlinks,
);
}
}
}
}
}

/// Maybe add `llvm-bitcode-linker` to the rustc sysroot.
pub fn maybe_install_llvm_bitcode_linker(
builder: &Builder<'_>,
builder_compiler: Compiler,
target: TargetSelection,
dst_dir: &Path,
dereference_symlinks: bool,
) {
if builder.config.llvm_bitcode_linker_enabled {
let llvm_bitcode_linker_exe = builder.ensure(tool::LlvmBitcodeLinker {
compiler: builder_compiler,
target,
extra_features: vec![],
});

builder.copy_link_internal(
&llvm_bitcode_linker_exe,
&dst_dir.join(llvm_bitcode_linker_exe.file_name().unwrap()),
dereference_symlinks,
);
}
}

/// Maybe add `wasm-component-ld` to the rustc sysroot.
pub fn maybe_install_wasm_component_ld(
builder: &Builder<'_>,
builder_compiler: Compiler,
target: TargetSelection,
dst_dir: &Path,
dereference_symlinks: bool,
) {
if builder.tool_enabled("wasm-component-ld") {
let wasm_component_ld_exe =
builder.ensure(tool::WasmComponentLd { compiler: builder_compiler, target });

builder.copy_link_internal(
&wasm_component_ld_exe,
&dst_dir.join(wasm_component_ld_exe.file_name().unwrap()),
dereference_symlinks,
);
}
}

#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct LlvmTools {
pub target: TargetSelection,
Expand Down

0 comments on commit adaff53

Please sign in to comment.