Auto merge of rust-lang#83640 - bjorn3:shared_metadata_reader, r=nagisa

Use the object crate for metadata reading

This allows sharing the metadata reader between cg_llvm, cg_clif and other codegen backends.

This is not currently useful for rlib reading with cg_spirv ([rust-gpu](https://github.com/EmbarkStudios/rust-gpu/)) as it uses tar rather than ar as .rlib format, but it is useful for dylib reading required for loading proc macros. (cc `@eddyb)`

The object crate is already trusted as dependency of libstd through backtrace. As far as I know it supports reading all object file formats used by targets for which we support rust dylibs with crate metadata, but I am not certain. If this happens to not be the case, I could keep using LLVM for reading dylib metadata.

Marked as WIP for a perf run and as it is based on rust-lang#83637.

Cargo.lock

@@ -3695,6 +3695,7 @@ dependencies = [
  "itertools 0.9.0",
  "jobserver",
  "libc",
+ "object",
  "pathdiff",
  "rustc_apfloat",
  "rustc_ast",

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -165,7 +165,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
     }
 
     fn metadata_loader(&self) -> Box<dyn MetadataLoader + Sync> {
-        Box::new(crate::metadata::CraneliftMetadataLoader)
+        Box::new(rustc_codegen_ssa::back::metadata::DefaultMetadataLoader)
     }
 
     fn provide(&self, _providers: &mut Providers) {}

compiler/rustc_codegen_cranelift/src/metadata.rs

@@ -1,73 +1,9 @@
-//! Reading and writing of the rustc metadata for rlibs and dylibs
+//! Writing of the rustc metadata for dylibs
 
-use std::fs::File;
-use std::path::Path;
-
-use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::memmap::Mmap;
-use rustc_data_structures::owning_ref::OwningRef;
-use rustc_data_structures::rustc_erase_owner;
-use rustc_data_structures::sync::MetadataRef;
-use rustc_middle::middle::cstore::MetadataLoader;
 use rustc_middle::ty::TyCtxt;
-use rustc_target::spec::Target;
 
 use crate::backend::WriteMetadata;
 
-/// The metadata loader used by cg_clif.
-///
-/// The metadata is stored in the same format as cg_llvm.
-///
-/// # Metadata location
-///
-/// <dl>
-/// <dt>rlib</dt>
-/// <dd>The metadata can be found in the `lib.rmeta` file inside of the ar archive.</dd>
-/// <dt>dylib</dt>
-/// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
-/// </dl>
-pub(crate) struct CraneliftMetadataLoader;
-
-fn load_metadata_with(
-    path: &Path,
-    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
-) -> Result<MetadataRef, String> {
-    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
-    let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?;
-    let metadata = OwningRef::new(data).try_map(f)?;
-    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
-}
-
-impl MetadataLoader for CraneliftMetadataLoader {
-    fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        load_metadata_with(path, |data| {
-            let archive = object::read::archive::ArchiveFile::parse(&*data)
-                .map_err(|e| format!("{:?}", e))?;
-
-            for entry_result in archive.members() {
-                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
-                if entry.name() == METADATA_FILENAME.as_bytes() {
-                    return Ok(entry.data());
-                }
-            }
-
-            Err("couldn't find metadata entry".to_string())
-        })
-    }
-
-    fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        use object::{Object, ObjectSection};
-
-        load_metadata_with(path, |data| {
-            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
-            file.section_by_name(".rustc")
-                .ok_or("no .rustc section")?
-                .data()
-                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
-        })
-    }
-}
-
 // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
 pub(crate) fn write_metadata<O: WriteMetadata>(tcx: TyCtxt<'_>, object: &mut O) {
     use snap::write::FrameEncoder;

compiler/rustc_codegen_llvm/src/base.rs

@@ -18,7 +18,6 @@ use crate::builder::Builder;
 use crate::common;
 use crate::context::CodegenCx;
 use crate::llvm;
-use crate::metadata;
 use crate::value::Value;
 
 use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
@@ -47,6 +46,22 @@ pub fn write_compressed_metadata<'tcx>(
     use snap::write::FrameEncoder;
     use std::io::Write;
 
+    // Historical note:
+    //
+    // When using link.exe it was seen that the section name `.note.rustc`
+    // was getting shortened to `.note.ru`, and according to the PE and COFF
+    // specification:
+    //
+    // > Executable images do not use a string table and do not support
+    // > section names longer than 8 characters
+    //
+    // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
+    //
+    // As a result, we choose a slightly shorter name! As to why
+    // `.note.rustc` works on MinGW, see
+    // https://github.com/llvm/llvm-project/blob/llvmorg-12.0.0/lld/COFF/Writer.cpp#L1190-L1197
+    let section_name = if tcx.sess.target.is_like_osx { "__DATA,.rustc" } else { ".rustc" };
+
     let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());
     let mut compressed = tcx.metadata_encoding_version();
     FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
@@ -59,7 +74,6 @@ pub fn write_compressed_metadata<'tcx>(
         unsafe { llvm::LLVMAddGlobal(metadata_llmod, common::val_ty(llconst), buf.as_ptr()) };
     unsafe {
         llvm::LLVMSetInitializer(llglobal, llconst);
-        let section_name = metadata::metadata_section_name(&tcx.sess.target);
         let name = SmallCStr::new(section_name);
         llvm::LLVMSetSection(llglobal, name.as_ptr());
 

compiler/rustc_codegen_llvm/src/lib.rs

@@ -69,7 +69,6 @@ pub mod llvm {
 }
 
 mod llvm_util;
-mod metadata;
 mod mono_item;
 mod type_;
 mod type_of;
@@ -251,7 +250,7 @@ impl CodegenBackend for LlvmCodegenBackend {
     }
 
     fn metadata_loader(&self) -> Box<MetadataLoaderDyn> {
-        Box::new(metadata::LlvmMetadataLoader)
+        Box::new(rustc_codegen_ssa::back::metadata::DefaultMetadataLoader)
     }
 
     fn provide(&self, _providers: &mut ty::query::Providers) {}

compiler/rustc_codegen_ssa/Cargo.toml

@@ -33,3 +33,8 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_target = { path = "../rustc_target" }
 rustc_session = { path = "../rustc_session" }
+
+[dependencies.object]
+version = "0.22.0"
+default-features = false
+features = ["read_core", "elf", "macho", "pe", "unaligned", "archive"]

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -0,0 +1,71 @@
+//! Reading of the rustc metadata for rlibs and dylibs
+
+use std::fs::File;
+use std::path::Path;
+
+use rustc_data_structures::memmap::Mmap;
+use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::rustc_erase_owner;
+use rustc_data_structures::sync::MetadataRef;
+use rustc_middle::middle::cstore::MetadataLoader;
+use rustc_target::spec::Target;
+
+use crate::METADATA_FILENAME;
+
+/// The default metadata loader. This is used by cg_llvm and cg_clif.
+///
+/// # Metadata location
+///
+/// <dl>
+/// <dt>rlib</dt>
+/// <dd>The metadata can be found in the `lib.rmeta` file inside of the ar archive.</dd>
+/// <dt>dylib</dt>
+/// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
+/// </dl>
+pub struct DefaultMetadataLoader;
+
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file =
+        File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?;
+    let data = unsafe { Mmap::map(file) }
+        .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e))?;
+    let metadata = OwningRef::new(data).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
+impl MetadataLoader for DefaultMetadataLoader {
+    fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
+        load_metadata_with(path, |data| {
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
+                .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?;
+
+            for entry_result in archive.members() {
+                let entry = entry_result
+                    .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
+            }
+
+            Err(format!("metadata not found in rlib '{}'", path.display()))
+        })
+    }
+
+    fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
+        use object::{Object, ObjectSection};
+
+        load_metadata_with(path, |data| {
+            let file = object::File::parse(&data)
+                .map_err(|e| format!("failed to parse dylib '{}': {}", path.display(), e))?;
+            file.section_by_name(".rustc")
+                .ok_or_else(|| format!("no .rustc section in '{}'", path.display()))?
+                .data()
+                .map_err(|e| {
+                    format!("failed to read .rustc section in '{}': {}", path.display(), e)
+                })
+        })
+    }
+}

compiler/rustc_codegen_ssa/src/back/mod.rs

@@ -3,6 +3,7 @@ pub mod command;
 pub mod link;
 pub mod linker;
 pub mod lto;
+pub mod metadata;
 pub mod rpath;
 pub mod symbol_export;
 pub mod write;

src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs

@@ -28,23 +28,11 @@ use rustc_target::spec::Target;
 use std::any::Any;
 use std::path::Path;
 
-pub struct NoLlvmMetadataLoader;
-
-impl MetadataLoader for NoLlvmMetadataLoader {
-    fn get_rlib_metadata(&self, _: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        unreachable!("some_crate.rs shouldn't depend on any external crates");
-    }
-
-    fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<MetadataRef, String> {
-        unreachable!("some_crate.rs shouldn't depend on any external crates");
-    }
-}
-
 struct TheBackend;
 
 impl CodegenBackend for TheBackend {
     fn metadata_loader(&self) -> Box<MetadataLoaderDyn> {
-        Box::new(NoLlvmMetadataLoader)
+        Box::new(rustc_codegen_ssa::back::metadata::DefaultMetadataLoader)
     }
 
     fn provide(&self, providers: &mut Providers) {}