diff --git a/test/e2e/fixtures/known_hosts b/test/e2e/fixtures/known_hosts
deleted file mode 100644
index 4b5bb2cfc..000000000
--- a/test/e2e/fixtures/known_hosts
+++ /dev/null
@@ -1,4 +0,0 @@
-# generated with "ssh-keyscan gitsrv"
-gitsrv ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMtNk10/31ytR+on/by5JrZ++ZpZ1OiUX9uplmNI7HJZvMINBALflVBIZ/KTESj8T2Jtzhh9I1JYHztpC4XdlVI=
-gitsrv ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2D9Zydkhs73och3XgvEkiDDdC2s5L5sIr4ZMn5/Ebcte3dqXPmwNXPmL/ljvHf55G3q863g/MfVdMHv7SZR7G690C9pJLfX02B4pVpwd9SFicQFc425kPV96wN/FaSHeux6wFDKwK17uYKzyG3FPjFY46nv14PlzLZid7ruJmGTb+2HsLpNFy/HzHmXL27aL5Cs0GwYOWKDR0UsovJlXa7XfKDNoVjwMhxmMhHbVAfLiVyQJrTj8o/27w9GcDAd6JkoY9vD6UNdQ3kks9tfC0ygEKBIrl2F0a7xMLLkSXQwJuZ0NgLvJZHqndGYhxGc3y0hWSpCwjs6f//wUztLP3
-gitsrv ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINrHJ/BT3DvC0Gqlqf/Pw0on/E4DB8/511Xnd8gwjggR
diff --git a/test/e2e/fixtures/kustom/base/gitsrv/gitsrv.yaml b/test/e2e/fixtures/kustom/base/gitsrv/gitsrv.yaml
index 764ec9d87..f4aec45e5 100644
--- a/test/e2e/fixtures/kustom/base/gitsrv/gitsrv.yaml
+++ b/test/e2e/fixtures/kustom/base/gitsrv/gitsrv.yaml
@@ -16,7 +16,7 @@ spec:
         name: gitsrv
     spec:
       containers:
-        - image: stefanprodan/gitsrv:0.1.3
+        - image: fluxcd/gitsrv:v1.0.0
           name: git
           env:
             - name: REPO
diff --git a/test/e2e/lib/env.bash b/test/e2e/lib/env.bash
index 21bbb7b2e..e5cd27738 100755
--- a/test/e2e/lib/env.bash
+++ b/test/e2e/lib/env.bash
@@ -6,11 +6,11 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 export ROOT_DIR
 export E2E_DIR="${ROOT_DIR}/test/e2e"
 export FIXTURES_DIR="${E2E_DIR}/fixtures"
-KNOWN_HOSTS=$(cat "${FIXTURES_DIR}/known_hosts")
-export KNOWN_HOSTS
 GITCONFIG=$(cat "${FIXTURES_DIR}/gitconfig")
 export GITCONFIG
 export HELM_VERSION=${HELM_VERSION}
+export GITSRV_VERSION=v1.0.0
+export GITSRV_KNOWN_HOSTS="${ROOT_DIR}/cache/known_hosts_${GITSRV_VERSION}"
 
 # Wire the test to the right cluster when tests are run in parallel
 if eval [ -n '$KUBECONFIG_SLOT_'"${BATS_JOB_SLOT}" ]; then
diff --git a/test/e2e/lib/install.bash b/test/e2e/lib/install.bash
index da8433500..d1433cc3d 100644
--- a/test/e2e/lib/install.bash
+++ b/test/e2e/lib/install.bash
@@ -28,6 +28,8 @@ function install_helm_operator_with_helm() {
     create_crds='false'
   fi
 
+  [ -f "${GITSRV_KNOWN_HOSTS}" ] || download_git_srv_known_hosts
+
   helm2 install --name helm-operator --wait \
     --tiller-namespace "${E2E_NAMESPACE}" \
     --namespace "${E2E_NAMESPACE}" \
@@ -40,7 +42,7 @@ function install_helm_operator_with_helm() {
     --set git.config.enabled=true \
     --set-string git.config.data="${GITCONFIG}" \
     --set git.ssh.secretName=flux-git-deploy \
-    --set-string git.ssh.known_hosts="${KNOWN_HOSTS}" \
+    --set-string git.ssh.known_hosts="$(cat "${GITSRV_KNOWN_HOSTS}")" \
     --set configureRepositories.enable=true \
     --set configureRepositories.repositories[0].name="stable" \
     --set configureRepositories.repositories[0].url="https://kubernetes-charts.storage.googleapis.com" \
@@ -60,6 +62,8 @@ function uninstall_helm_operator_with_helm() {
 }
 
 function install_git_srv() {
+  [ -f "${GITSRV_KNOWN_HOSTS}" ] || download_git_srv_known_hosts
+
   local external_access_result_var=${1}
   local kustomization_dir=${2:-base/gitsrv}
   local gen_dir
@@ -68,7 +72,7 @@ function install_git_srv() {
   ssh-keygen -t rsa -N "" -f "$gen_dir/id_rsa"
   kubectl create secret generic flux-git-deploy \
     --namespace="${E2E_NAMESPACE}" \
-    --from-file="${FIXTURES_DIR}/known_hosts" \
+    --from-file="${GITSRV_KNOWN_HOSTS}" \
     --from-file="$gen_dir/id_rsa" \
     --from-file=identity="$gen_dir/id_rsa" \
     --from-file="$gen_dir/id_rsa.pub"
@@ -92,6 +96,11 @@ function install_git_srv() {
   fi
 }
 
+function download_git_srv_known_hosts() {
+  mkdir -p "$(dirname "${GITSRV_KNOWN_HOSTS}")"
+  curl -sL "https://github.com/fluxcd/gitsrv/releases/download/${GITSRV_VERSION}/known_hosts.txt" > "${GITSRV_KNOWN_HOSTS}"
+}
+
 function uninstall_git_srv() {
   local kustomization_dir=${1:-base/gitsrv}