diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index bb6425d5cb2..4ad54fae8fa 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -213,7 +213,7 @@ jobs:
     needs: wheel
     strategy:
       matrix:
-        framework: ["numpy", "pytorch", "tensorflow", "huggingface", "jax", "sklearn"]
+        framework: ["numpy", "pytorch", "tensorflow", "jax", "sklearn"]
 
     name: Template / ${{ matrix.framework }}
 
@@ -234,11 +234,6 @@ jobs:
           flwr new tmp-${{ matrix.framework }} --framework ${{ matrix.framework }} --username gh_ci
           cd tmp-${{ matrix.framework }}
           pip install .
-      - name: Cache Datasets
-        uses: actions/cache@v4
-        with:
-          path: "~/.cache/huggingface/datasets"
-          key: ${{ matrix.framework }}-template-datasets
       - name: Run project
         run: |
           cd tmp-${{ matrix.framework }}