Skip to content

Commit

Permalink
feat(SourceId): use stable hash from rustc-stable-hash
Browse files Browse the repository at this point in the history
This helps `-Ztrim-paths` build a stable cross-platform path for the
registry and git sources. Sources files then can be found from the same
path when debugging.

It also helps cache registry index all at once for all platforms,
for example the use case in rust-lang#14795
(despite they should use `cargo vendor` instead IMO).

Some caveats:

* Newer cargo will need to re-download files for global caches
  (index files, git/registry sources).
  The old cache is still kept and used when running with older cargoes.
* Windows is not really covered by the "cross-platform` hash,
  because path prefix components like `C:` are always there.
  That means hashes of some sources kind,
  like local registry and local path,
  are not going to be real cross-platform stable.

There might be hash collisions if you have two registries under the same
domain. This won't happen to crates.io, as the infra would have to
intentionally put another registry on index.crates.io to collide.
We don't consider this is an actual threat model, so we are not going to
use any cryptographically secure hash algorithm like BLAKE3.

See also <rust-lang#13171 (comment)>
  • Loading branch information
weihanglo committed Dec 10, 2024
1 parent 674e609 commit b74cc26
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 60 deletions.
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ rand = "0.8.5"
regex = "1.10.5"
rusqlite = { version = "0.32.0", features = ["bundled"] }
rustc-hash = "2.0.0"
rustc-stable-hash = "0.1.1"
rustfix = { version = "0.9.0", path = "crates/rustfix" }
same-file = "1.0.6"
schemars = "0.8.21"
Expand Down Expand Up @@ -194,6 +195,7 @@ rand.workspace = true
regex.workspace = true
rusqlite.workspace = true
rustc-hash.workspace = true
rustc-stable-hash.workspace = true
rustfix.workspace = true
same-file.workspace = true
semver.workspace = true
Expand Down
91 changes: 55 additions & 36 deletions src/cargo/core/source_id.rs
Original file line number Diff line number Diff line change
Expand Up @@ -782,70 +782,89 @@ mod tests {
// Otherwise please just leave a comment in your PR as to why the hash value is
// changing and why the old value can't be easily preserved.
//
// The hash value depends on endianness and bit-width, so we only run this test on
// little-endian 64-bit CPUs (such as x86-64 and ARM64) where it matches the
// well-known value.
// The hash value should be stable across platforms, and doesn't depend on
// endianness and bit-width. One caveat is that absolute paths on Windows
// are inherently different than on Unix-like platforms. Unless we omit or
// strip the prefix components (e.g. `C:`), there is not way to have a true
// cross-platform stable hash for absolute paths.
#[test]
#[cfg(all(target_endian = "little", target_pointer_width = "64"))]
fn test_cratesio_hash() {
let gctx = GlobalContext::default().unwrap();
let crates_io = SourceId::crates_io(&gctx).unwrap();
assert_eq!(crate::util::hex::short_hash(&crates_io), "1ecc6299db9ec823");
}

// See the comment in `test_cratesio_hash`.
//
// Only test on non-Windows as paths on Windows will get different hashes.
#[test]
#[cfg(all(target_endian = "little", target_pointer_width = "64", not(windows)))]
fn test_stable_hash() {
use std::hash::Hasher;
use std::path::Path;

use crate::util::StableHasher;

#[cfg(not(windows))]
let ws_root = Path::new("/tmp/ws");
#[cfg(windows)]
let ws_root = Path::new(r"C:\\tmp\ws");

let gen_hash = |source_id: SourceId| {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
source_id.stable_hash(Path::new("/tmp/ws"), &mut hasher);
hasher.finish()
let mut hasher = StableHasher::new();
source_id.stable_hash(ws_root, &mut hasher);
Hasher::finish(&hasher)
};

let source_id = SourceId::crates_io(&GlobalContext::default().unwrap()).unwrap();
assert_eq!(gen_hash(source_id), 7062945687441624357);
assert_eq!(crate::util::hex::short_hash(&source_id), "25cdd57fae9f0462");

let url = "https://my-crates.io".into_url().unwrap();
let source_id = SourceId::for_registry(&url).unwrap();
assert_eq!(gen_hash(source_id), 18108075011063494626);
assert_eq!(crate::util::hex::short_hash(&source_id), "fb60813d6cb8df79");
assert_eq!(gen_hash(source_id), 8310250053664888498);
assert_eq!(crate::util::hex::short_hash(&source_id), "b2d65deb64f05373");

let url = "https://your-crates.io".into_url().unwrap();
let source_id = SourceId::for_alt_registry(&url, "alt").unwrap();
assert_eq!(gen_hash(source_id), 12862859764592646184);
assert_eq!(crate::util::hex::short_hash(&source_id), "09c10fd0cbd74bce");
assert_eq!(gen_hash(source_id), 14149534903000258933);
assert_eq!(crate::util::hex::short_hash(&source_id), "755952de063f5dc4");

let url = "sparse+https://my-crates.io".into_url().unwrap();
let source_id = SourceId::for_registry(&url).unwrap();
assert_eq!(gen_hash(source_id), 8763561830438022424);
assert_eq!(crate::util::hex::short_hash(&source_id), "d1ea0d96f6f759b5");
assert_eq!(gen_hash(source_id), 16249512552851930162);
assert_eq!(crate::util::hex::short_hash(&source_id), "327cfdbd92dd81e1");

let url = "sparse+https://your-crates.io".into_url().unwrap();
let source_id = SourceId::for_alt_registry(&url, "alt").unwrap();
assert_eq!(gen_hash(source_id), 5159702466575482972);
assert_eq!(crate::util::hex::short_hash(&source_id), "135d23074253cb78");
assert_eq!(gen_hash(source_id), 6156697384053352292);
assert_eq!(crate::util::hex::short_hash(&source_id), "64a713b6a6fb7055");

let url = "file:///tmp/ws/crate".into_url().unwrap();
let source_id = SourceId::for_git(&url, GitReference::DefaultBranch).unwrap();
assert_eq!(gen_hash(source_id), 15332537265078583985);
assert_eq!(crate::util::hex::short_hash(&source_id), "73a808694abda756");

let path = Path::new("/tmp/ws/crate");
assert_eq!(gen_hash(source_id), 473480029881867801);
assert_eq!(crate::util::hex::short_hash(&source_id), "199e591d94239206");

let path = &ws_root.join("crate");
let source_id = SourceId::for_local_registry(path).unwrap();
assert_eq!(gen_hash(source_id), 18446533307730842837);
assert_eq!(crate::util::hex::short_hash(&source_id), "52a84cc73f6fd48b");
#[cfg(not(windows))]
{
assert_eq!(gen_hash(source_id), 11515846423845066584);
assert_eq!(crate::util::hex::short_hash(&source_id), "58d73c154f81d09f");
}
#[cfg(windows)]
{
assert_eq!(gen_hash(source_id), 6146331155906064276);
assert_eq!(crate::util::hex::short_hash(&source_id), "946fb2239f274c55");
}

let source_id = SourceId::for_path(path).unwrap();
assert_eq!(gen_hash(source_id), 8764714075439899829);
assert_eq!(crate::util::hex::short_hash(&source_id), "e1ddd48578620fc1");
assert_eq!(gen_hash(source_id), 215644081443634269);
#[cfg(not(windows))]
assert_eq!(crate::util::hex::short_hash(&source_id), "64bace89c92b101f");
#[cfg(windows)]
assert_eq!(crate::util::hex::short_hash(&source_id), "01e1e6c391813fb6");

let source_id = SourceId::for_directory(path).unwrap();
assert_eq!(gen_hash(source_id), 17459999773908528552);
assert_eq!(crate::util::hex::short_hash(&source_id), "6568fe2c2fab5bfe");
#[cfg(not(windows))]
{
assert_eq!(gen_hash(source_id), 6127590343904940368);
assert_eq!(crate::util::hex::short_hash(&source_id), "505191d1f3920955");
}
#[cfg(windows)]
{
assert_eq!(gen_hash(source_id), 10423446877655960172);
assert_eq!(crate::util::hex::short_hash(&source_id), "6c8ad69db585a790");
}
}

#[test]
Expand Down
26 changes: 2 additions & 24 deletions src/cargo/util/hasher.rs
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
//! Implementation of a hasher that produces the same values across releases.
//!
//! The hasher should be fast and have a low chance of collisions (but is not
//! sufficient for cryptographic purposes).
#![allow(deprecated)]
//! A hasher that produces the same values across releases and platforms.
use std::hash::{Hasher, SipHasher};

#[derive(Clone)]
pub struct StableHasher(SipHasher);

impl StableHasher {
pub fn new() -> StableHasher {
StableHasher(SipHasher::new())
}
}

impl Hasher for StableHasher {
fn finish(&self) -> u64 {
self.0.finish()
}
fn write(&mut self, bytes: &[u8]) {
self.0.write(bytes)
}
}
pub use rustc_stable_hash::StableSipHasher128 as StableHasher;

0 comments on commit b74cc26

Please sign in to comment.