diff --git a/md5/Cargo.toml b/md5/Cargo.toml
index 594865aff..9b71d348a 100644
--- a/md5/Cargo.toml
+++ b/md5/Cargo.toml
@@ -16,6 +16,8 @@ name = "md5"
 
 [dependencies]
 digest = "0.10.3"
+
+[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
 md5-asm = { version = "0.5", optional = true }
 
 [dev-dependencies]
diff --git a/md5/src/lib.rs b/md5/src/lib.rs
index 0dcb14fdc..3a9970161 100644
--- a/md5/src/lib.rs
+++ b/md5/src/lib.rs
@@ -31,10 +31,10 @@
 )]
 #![warn(missing_docs, rust_2018_idioms)]
 
-#[cfg(feature = "asm")]
+#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))]
 extern crate md5_asm as compress;
 
-#[cfg(not(feature = "asm"))]
+#[cfg(not(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64"))))]
 mod compress;
 
 pub use digest::{self, Digest};
diff --git a/sha1/Cargo.toml b/sha1/Cargo.toml
index 6627712af..1878d3e22 100644
--- a/sha1/Cargo.toml
+++ b/sha1/Cargo.toml
@@ -15,10 +15,10 @@ categories = ["cryptography", "no-std"]
 [dependencies]
 digest = "0.10.3"
 cfg-if = "1.0"
-sha1-asm = { version = "0.5", optional = true }
 
 [target.'cfg(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64"))'.dependencies]
 cpufeatures = "0.2"
+sha1-asm = { version = "0.5", optional = true }
 
 [dev-dependencies]
 digest = { version = "0.10.3", features = ["dev"] }
diff --git a/sha2/Cargo.toml b/sha2/Cargo.toml
index cc4c783c8..4f563f6ea 100644
--- a/sha2/Cargo.toml
+++ b/sha2/Cargo.toml
@@ -17,10 +17,10 @@ categories = ["cryptography", "no-std"]
 [dependencies]
 digest = "0.10.3"
 cfg-if = "1.0"
-sha2-asm = { version = "0.6.1", optional = true }
 
 [target.'cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))'.dependencies]
 cpufeatures = "0.2"
+sha2-asm = { version = "0.6.1", optional = true }
 
 [dev-dependencies]
 digest = { version = "0.10.3", features = ["dev"] }
diff --git a/whirlpool/Cargo.toml b/whirlpool/Cargo.toml
index 5844b701f..573b28e44 100644
--- a/whirlpool/Cargo.toml
+++ b/whirlpool/Cargo.toml
@@ -13,6 +13,8 @@ categories = ["cryptography", "no-std"]
 
 [dependencies]
 digest = "0.10.3"
+
+[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
 whirlpool-asm = { version = "0.6", optional = true}
 
 [dev-dependencies]
diff --git a/whirlpool/src/lib.rs b/whirlpool/src/lib.rs
index 996c9b797..62f863ea4 100644
--- a/whirlpool/src/lib.rs
+++ b/whirlpool/src/lib.rs
@@ -45,10 +45,10 @@
 
 pub use digest::{self, Digest};
 
-#[cfg(not(feature = "asm"))]
+#[cfg(not(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64"))))]
 mod compress;
 
-#[cfg(feature = "asm")]
+#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))]
 use whirlpool_asm as compress;
 
 use compress::compress;