diff --git a/.github/workflows/build-arm64-image.yaml b/.github/workflows/build-arm64-image.yaml
index 079626b4205..4a1428c7ab2 100644
--- a/.github/workflows/build-arm64-image.yaml
+++ b/.github/workflows/build-arm64-image.yaml
@@ -18,10 +18,38 @@ jobs:
         with:
           go-version: ^1.14
         id: go
+
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+
       - name: Docker Buildx
         uses: crazy-max/ghaction-docker-buildx@v1.4.0
+
+      - name: Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: /home/runner/.cache/go-build
+          key: ${{ runner.os }}-arm64-go-build-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-arm64-go-build-
+
+      - name: Go Mod Cache
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-arm64-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-arm64-go-
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-arm64-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-arm64-buildx-
+
       - name: Build
         run: |
           go get -u github.com/securego/gosec/cmd/gosec
diff --git a/Makefile b/Makefile
index 90b6b4a2862..310e3034010 100644
--- a/Makefile
+++ b/Makefile
@@ -38,7 +38,7 @@ release: lint build-go
 	docker buildx build --cache-from "type=local,src=/tmp/.buildx-cache" --cache-to "type=local,dest=/tmp/.buildx-cache" --platform linux/amd64 --build-arg ARCH=amd64 --build-arg RPM_ARCH=x86_64 -t ${REGISTRY}/kube-ovn:${RELEASE_TAG} -o type=docker -f dist/images/Dockerfile dist/images/
 
 release-arm: lint build-go-arm
-	docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 --build-arg RPM_ARCH=aarch64 -t ${REGISTRY}/kube-ovn:${RELEASE_TAG} -o type=docker -f dist/images/Dockerfile dist/images/
+	docker buildx build --cache-from "type=local,src=/tmp/.buildx-cache" --cache-to "type=local,dest=/tmp/.buildx-cache" --platform linux/arm64 --build-arg ARCH=arm64 --build-arg RPM_ARCH=aarch64 -t ${REGISTRY}/kube-ovn:${RELEASE_TAG} -o type=docker -f dist/images/Dockerfile dist/images/
 
 tar:
 	docker save ${REGISTRY}/kube-ovn:${RELEASE_TAG} > image.tar