diff --git a/.dockerignore b/.dockerignore
index 4cf045c0ce..56eea687a2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -15,3 +15,9 @@ LICENSE
 rustfmt.toml
 _config.yml
 .lycheeignore
+
+# Ignore rust toolchain files as they would needlessly fetch the toolchain.
+# But the image already contains a sufficient toolchain.
+# Ignore both the latest naming and the legacy naming
+rust-toolchain.toml
+rust-toolchain