diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e5ea07a54e7..69e98a72310 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -119,13 +119,14 @@ jobs:
   build-windows:
     needs: check-changes
     if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
-    runs-on: [windows-2019]
+    runs-on: [ubuntu-latest]
     steps:
     - uses: actions/checkout@v4
       with:
         show-progress: false
     - name: Build Antrea Windows Docker image
-      run: make build-windows
+      if: ${{ github.repository != 'antrea-io/antrea' || github.event_name != 'push' || github.ref != 'refs/heads/main' }}
+      run: ./hack/build-antrea-windows-all.sh --pull
     - name: Push Antrea Windows Docker image to registry
       if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       env:
@@ -133,7 +134,7 @@ jobs:
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/antrea-windows:latest
+        ./hack/build-antrea-windows-all.sh --pull --push --push-base-images
       shell: bash
 
   build-antrea-mc-controller:
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
index f24cf4a4336..97cc1d2e618 100644
--- a/.github/workflows/build_tag.yml
+++ b/.github/workflows/build_tag.yml
@@ -66,7 +66,7 @@ jobs:
           docker push antrea/antrea-controller-ubi:"${VERSION}"
 
   build-windows:
-    runs-on: [windows-2019]
+    runs-on: [ubuntu-latest]
     needs: get-version
     steps:
     - uses: actions/checkout@v4
@@ -78,9 +78,8 @@ jobs:
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         VERSION: ${{ needs.get-version.outputs.version }}
       run: |
-        make build-windows
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-        docker push antrea/antrea-windows:"${VERSION}"
+        ./hack/build-antrea-windows-all.sh --pull --push
       shell: bash
 
   build-antrea-mc-controller:
diff --git a/.github/workflows/docker_update_base_windows.yml b/.github/workflows/docker_update_base_windows.yml
deleted file mode 100644
index 393521fc34d..00000000000
--- a/.github/workflows/docker_update_base_windows.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-# Anyone with write permissions to the antrea-io/antrea Github repository can
-# trigger this workflow manually, but please check with a maintainer first. The
-# workflow will build and push the antrea/base-windows image.
-name: Manually update antrea/base-windows Docker image
-
-on:
-  workflow_dispatch:
-    inputs:
-      antrea-repository:
-        description: 'The Antrea repository to check-out; it will typically be a personal Antrea fork such as "<your-github-user>/antrea"'
-        required: true
-        type: string
-      antrea-ref:
-        description: 'The Git ref to use when checking-out the Antrea repository, usually a branch or tag name'
-        required: true
-        type: string
-      push:
-        description: 'Whether to push built base images to the Docker registry'
-        required: false
-        default: true
-        type: boolean
-
-jobs:
-  build:
-    runs-on: windows-2019
-    steps:
-    - name: Check-out code
-      uses: actions/checkout@v4
-      with:
-        repository: ${{ inputs.antrea-repository }}
-        ref: ${{ inputs.antrea-ref }}
-        show-progress: false
-    - name: Login to DockerHub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Build and push Docker images
-      if: ${{ inputs.push }}
-      run: |
-        ./hack/build-antrea-windows-all.sh --pull --push-base-images
-      shell: bash
-    - name: Build Docker images without pushing
-      if: ${{ !inputs.push }}
-      run: |
-        ./hack/build-antrea-windows-all.sh --pull
-      shell: bash
diff --git a/Makefile b/Makefile
index 4c420f5c961..062c54bde49 100644
--- a/Makefile
+++ b/Makefile
@@ -19,11 +19,7 @@ ANTCTL_BINARY_NAME ?= antctl
 OVS_VERSION        := $(shell head -n 1 build/images/deps/ovs-version)
 GO_VERSION         := $(shell head -n 1 build/images/deps/go-version)
 CNI_BINARIES_VERSION := $(shell head -n 1 build/images/deps/cni-binaries-version)
-NANOSERVER_VERSION := $(shell head -n 1 build/images/deps/nanoserver-version)
 BUILD_TAG          := $(shell build/images/build-tag.sh)
-WIN_BUILD_TAG      := $(shell echo $(GO_VERSION) $(CNI_BINARIES_VERSION) $(NANOSERVER_VERSION)|md5sum|head -c 10)
-WIN_OVS_VERSION	   := $(shell head -n 1 build/images/deps/ovs-version-windows)
-WIN_BUILD_OVS_TAG  := $(NANOSERVER_VERSION)-$(WIN_OVS_VERSION)
 GIT_HOOKS          := $(shell find hack/git_client_side_hooks -type f -print)
 DOCKER_NETWORK     ?= default
 TRIVY_TARGET_IMAGE ?=
@@ -39,15 +35,9 @@ endif
 ifneq ($(NO_CACHE),)
 	DOCKER_BUILD_ARGS += --no-cache
 endif
-WIN_BUILD_ARGS := DOCKER_BUILD_ARGS
 DOCKER_BUILD_ARGS += --build-arg OVS_VERSION=$(OVS_VERSION)
 DOCKER_BUILD_ARGS += --build-arg GO_VERSION=$(GO_VERSION)
 DOCKER_BUILD_ARGS += --build-arg BUILD_TAG=$(BUILD_TAG)
-WIN_BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION)
-WIN_BUILD_ARGS += --build-arg CNI_BINARIES_VERSION=$(CNI_BINARIES_VERSION)
-WIN_BUILD_ARGS += --build-arg NANOSERVER_VERSION=$(NANOSERVER_VERSION)
-WIN_BUILD_ARGS += --build-arg WIN_BUILD_TAG=$(WIN_BUILD_TAG)
-WIN_BUILD_ARGS += --build-arg WIN_BUILD_OVS_TAG=$(WIN_BUILD_OVS_TAG)
 
 export CGO_ENABLED
 
@@ -62,6 +52,14 @@ UNAME_S := $(shell uname -s)
 USERID  := $(shell id -u)
 GRPID   := $(shell id -g)
 
+WINDOWS_BUILD_OPTIONS :=
+ifeq ($(NO_PULL),)
+	WINDOWS_BUILD_OPTIONS += --pull
+endif
+ifeq ($(RELEASE_STATUS),released)
+	WINDOWS_BUILD_OPTIONS += --agent-tag ${DOCKER_IMG_VERSION}
+endif
+
 .PHONY: install-hooks
 install-hooks:
 	@echo "===> Copying Antrea git hooks to local <==="
@@ -383,8 +381,12 @@ build-controller-ubi:
 .PHONY: build-windows
 build-windows:
 	@echo "===> Building Antrea bins and antrea/antrea-windows Docker image <==="
-	docker build -t antrea/antrea-windows:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.build.windows --network $(DOCKER_NETWORK) $(WIN_BUILD_ARGS) .
-	docker tag antrea/antrea-windows:$(DOCKER_IMG_VERSION) antrea/antrea-windows
+	$(CURDIR)/build/images/build-windows.sh ${WINDOWS_BUILD_OPTIONS}
+
+.PHONY: build-and-push-windows
+build-and-push-windows:
+	@echo "===> Building Antrea bins and antrea/antrea-windows Docker image and pushing to registry <==="
+	$(CURDIR)/build/images/build-windows.sh --push ${WINDOWS_BUILD_OPTIONS}
 
 .PHONY: build-ubuntu-coverage
 build-ubuntu-coverage: build-controller-ubuntu-coverage build-agent-ubuntu-coverage
diff --git a/build/images/Dockerfile.build.windows b/build/images/Dockerfile.build.windows
index 3ab92df561c..24cab356715 100644
--- a/build/images/Dockerfile.build.windows
+++ b/build/images/Dockerfile.build.windows
@@ -12,45 +12,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG WIN_BUILD_TAG
-ARG NANOSERVER_VERSION
-ARG WIN_BUILD_OVS_TAG
+ARG GO_VERSION
+ARG OVS_VERSION
 
-FROM antrea/base-windows:${WIN_BUILD_TAG} as antrea-build-windows
+FROM --platform=linux/amd64 golang:${GO_VERSION} as antrea-build-windows
+ARG CNI_BINARIES_VERSION
+ENV CNI_PLUGINS="./host-local.exe"
 
 WORKDIR /antrea
 
-COPY go.mod /antrea/go.mod
+ADD https://github.com/containernetworking/plugins/releases/download/${CNI_BINARIES_VERSION}/cni-plugins-windows-amd64-${CNI_BINARIES_VERSION}.tgz cni-plugins-windows-amd64.tgz
+RUN mkdir -p /go/k/antrea/cni && \
+    tar -xzf cni-plugins-windows-amd64.tgz -C /go/k/antrea/cni ${CNI_PLUGINS} && \
+    rm cni-plugins-windows-amd64.tgz
 
-RUN go mod download
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=bind,source=go.sum,target=go.sum \
+    --mount=type=bind,source=go.mod,target=go.mod \
+    go mod download
 
 COPY . /antrea
 
-RUN sh -c 'make windows-bin'
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make windows-bin
 
-FROM antrea/windows-ovs:${WIN_BUILD_OVS_TAG} as windows-ovs
+RUN mkdir -p /go/k/antrea/bin && \
+    cp /antrea/bin/antrea-agent.exe /go/k/antrea/bin/ && \
+    cp /antrea/bin/antctl.exe /go/k/antrea/bin/ && \
+    cp /antrea/bin/antrea-cni.exe /go/k/antrea/cni/antrea.exe
 
-FROM mcr.microsoft.com/powershell:lts-nanoserver-${NANOSERVER_VERSION}
-SHELL ["pwsh", "-NoLogo", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+FROM antrea/windows-ovs:${OVS_VERSION} as antrea-ovs
 
-LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
-LABEL description="A Docker image to deploy the Antrea CNI."
+FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0
+COPY --from=antrea-build-windows /go/k /k
+COPY --from=antrea-ovs /openvswitch /openvswitch
 
-USER ContainerAdministrator
-
-RUN mkdir -Force C:\k\antrea\bin
-COPY --from=antrea-build-windows  /opt/cni/bin /k/antrea/cni
-COPY --from=antrea-build-windows  /antrea/build/images/scripts/Install-WindowsCNI.ps1 /k/antrea/
-COPY --from=antrea-build-windows  /antrea/bin/antrea-agent.exe /k/antrea/bin/
-COPY --from=antrea-build-windows  /antrea/bin/antctl.exe /k/antrea/bin/antctl.exe
-COPY --from=antrea-build-windows  /antrea/bin/antrea-cni.exe /k/antrea/cni/antrea.exe
-
-RUN mkdir C:\k\antrea\utils
-COPY --from=antrea-build-windows /wins/wins.exe /k/antrea/utils/wins.exe
-
-COPY --from=windows-ovs /Windows/System32/vcruntime140.dll /Windows/System32/
-COPY --from=windows-ovs /Windows/System32/libeay32.dll /Windows/System32/
-COPY --from=windows-ovs /Windows/System32/ssleay32.dll /Windows/System32/
-
-RUN mkdir C:\openvswitch
-COPY --from=windows-ovs /openvswitch /openvswitch/
+ENV PATH="C:\Windows\system32;C:\Windows;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\Windows\System32\wbem;"
diff --git a/build/images/base-windows/Dockerfile b/build/images/base-windows/Dockerfile
deleted file mode 100644
index b0802d33e19..00000000000
--- a/build/images/base-windows/Dockerfile
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2022 Antrea Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GO_VERSION
-ARG NANOSERVER_VERSION
-
-FROM mcr.microsoft.com/windows/servercore:${NANOSERVER_VERSION} as windows-utility-base
-
-ARG CNI_BINARIES_VERSION
-
-SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
-
-ENV CNI_BINARIES_VERSION ${CNI_BINARIES_VERSION}
-ENV CNI_PLUGINS="./host-local.exe"
-
-WORKDIR /
-
-RUN mkdir -Force C:\opt\cni\bin
-
-RUN Invoke-WebRequest -UseBasicParsing -Uri https://github.com/containernetworking/plugins/releases/download/${env:CNI_BINARIES_VERSION}/cni-plugins-windows-amd64-${env:CNI_BINARIES_VERSION}.tgz -OutFile cni-plugins-windows-amd64-${env:CNI_BINARIES_VERSION}.tgz; \
-    tar -xzf cni-plugins-windows-amd64-${env:CNI_BINARIES_VERSION}.tgz  -C C:\opt\cni\bin ${env:CNI_PLUGINS}; \
-    rm cni-plugins-windows-amd64-${env:CNI_BINARIES_VERSION}.tgz
-
-# Install 7zip, git-for-windows, mingw64 to support "make tool"
-RUN Invoke-WebRequest -UseBasicParsing -Uri https://www.7-zip.org/a/7z2107-x64.exe -OutFile 7z2107-x64.exe; \
-    cmd /c start /wait 7z2107-x64.exe /S; \
-    del 7z2107-x64.exe;  $env:Path = $env:Path+';C:/Program Files/7-Zip'; \
-    # Force the User-Agent to "wget" so that we are redirected directly to a download mirror. With
-    # the default, Sourceforge may treat us as a browser and the download would fail.
-    Invoke-WebRequest -UseBasicParsing -UserAgent "Wget" -Uri https://downloads.sourceforge.net/project/mingw-w64/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/8.1.0/threads-posix/seh/x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z -OutFile mingw.7z; \
-    7z x mingw.7z; cp c:/mingw64/bin/mingw32-make.exe c:/mingw64/bin/make.exe; \
-    Invoke-WebRequest -UseBasicParsing -Uri https://github.com/git-for-windows/git/releases/download/v2.35.1.windows.2/PortableGit-2.35.1.2-64-bit.7z.exe -OutFile git.exe; \
-    7z x git.exe -oC:\git; \
-    mkdir C:\wins; \
-    Invoke-WebRequest -UseBasicParsing -Uri https://github.com/rancher/wins/releases/download/v0.0.4/wins.exe -OutFile C:/wins/wins.exe
-
-FROM golang:${GO_VERSION}-nanoserver as windows-golang
-
-FROM mcr.microsoft.com/windows/nanoserver:${NANOSERVER_VERSION}
-
-SHELL ["cmd", "/S", "/C"]
-
-ENV GOPATH="C:\go"
-
-USER ContainerAdministrator
-
-COPY --from=windows-utility-base ["C:\\\\opt\\\\cni\\\\bin","C:\\\\opt\\\\cni\\\\bin"]
-COPY --from=windows-utility-base ["C:\\\\Program Files\\\\7-Zip","C:\\\\Program Files\\\\7-Zip"]
-COPY --from=windows-utility-base ["C:\\\\git","C:\\\\git"]
-COPY --from=windows-utility-base ["C:\\\\mingw64","C:\\\\mingw64"]
-COPY --from=windows-utility-base ["C:\\\\wins","C:\\\\wins"]
-COPY --from=windows-golang ["C:\\\\Program Files\\\\Go", "C:\\\\Program Files\\\\Go"]
-
-RUN setx /m PATH "C:\Program Files\7-Zip;C:\git\bin;C:\git\usr\bin;C:\mingw64\bin;C:\Program Files\Go\bin;%GOPATH\bin%;%PATH%"
diff --git a/build/images/base-windows/README.md b/build/images/base-windows/README.md
deleted file mode 100644
index 4f617779c55..00000000000
--- a/build/images/base-windows/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# images/base-windows
-
-This Docker image includes download libraries for building windows image.
-
-If you need to build a new version of the image and push it to Dockerhub, you
-can run the following:
-
-```bash
-cd build/images/base-windows
-GO_VERSION=$(head -n 1 ../deps/go-version)
-CNI_BINARIES_VERSION=$(head -n 1 ../deps/cni-binaries-version)
-NANOSERVER_VERSION=$(head -n 1 ../deps/nanoserver-version)
-WIN_BUILD_TAG=$(echo $GO_VERSION $CNI_BINARIES_VERSION $NANOSERVER_VERSION| md5sum| head -c 10)
-docker build -t antrea/base-windows:$WIN_BUILD_TAG --build-arg GO_VERSION=$GO_VERSION --build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION --build-arg NANOSERVER_VERSION=$NANOSERVER_VERSION .
-docker push antrea/base-windows:$WIN_BUILD_TAG
-```
-
-The `docker push` command will fail if you do not have permission to push to the
-`antrea` Dockerhub repository.
-
-However, the easiest way to push a new image on Dockerhub is to run the
-`Manually update antrea/base-windows Docker image` Github workflow. Only
-contributors with `write` access to the antrea-io/antrea Github repository can
-trigger the workflow. If you need to update the image, please check with a
-maintainer first.
diff --git a/build/images/build-utils.sh b/build/images/build-utils.sh
index 986801d1906..414211debbc 100644
--- a/build/images/build-utils.sh
+++ b/build/images/build-utils.sh
@@ -41,3 +41,39 @@ function check_docker_build_driver() {
     fi
     return 0
 }
+
+function switch_windows_buildx() {
+    local windows_buildx_name="windows-img-builder"
+    original_buildx_instance=$(docker buildx inspect | grep '^Name:' | awk '{print $2}' | head -n 1)
+    if [ "$original_buildx_instance" = "${windows_buildx_name}" ]; then
+        return
+    fi
+    trap 'docker buildx use --default ${original_buildx_instance}' EXIT
+    set +e
+    docker buildx ls | grep "${windows_buildx_name}" > /dev/null 2>&1
+    if [ $? -eq 0 ] ; then
+        docker buildx use --builder windows/amd64 "${windows_buildx_name}"
+    else
+        docker buildx create --name "${windows_buildx_name}" --use --platform windows/amd64
+    fi
+    set -e
+}
+
+function docker_build_and_push_windows() {
+    local image="$1"
+    local dockerfile="$2"
+    local build_args="$3"
+    local build_tag="$4"
+    local push=$5
+    local pull_option="$6"
+
+    switch_windows_buildx
+    if $push; then
+        output="type=registry"
+    else
+        local_file=$(echo "${image}" | awk -F'/' '{print $NF}')
+        output="type=docker,dest=./${local_file}.tar"
+    fi
+
+    docker buildx build --platform windows/amd64 -o ${output} -t ${image}:${build_tag} ${pull_option} ${build_args} -f $dockerfile .
+}
diff --git a/build/images/build-windows.sh b/build/images/build-windows.sh
new file mode 100755
index 00000000000..baf3a1e5e1f
--- /dev/null
+++ b/build/images/build-windows.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Antrea Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a very simple script that builds the base image for Antrea and pushes it to
+# the Antrea Dockerhub (https://hub.docker.com/u/antrea). The image is tagged with the OVS version.
+
+set -eo pipefail
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+source $THIS_DIR/build-utils.sh
+
+_usage="Usage: $0 [--pull] [--push] [--agent-tag]
+Build the antrea base image.
+        --pull                 Always attempt to pull a newer version of Window OVS image.
+        --push                 Push the built image to the registry
+        --agent-tag            Antrea Agent image tag"
+
+PULL_OPTION=""
+PUSH=false
+BUILD_TAG="latest"
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    --pull)
+    PULL_OPTION="--pull"
+    shift
+    ;;
+    --push)
+    PUSH=true
+    shift
+    ;;
+    --agent-tag)
+    BUILD_TAG="$2"
+    shift 2
+    ;;
+    -h|--help)
+    print_usage
+    exit 0
+    ;;
+    *)    # unknown option
+    echoerr "Unknown option $1"
+    exit 1
+    ;;
+esac
+done
+
+pushd $THIS_DIR > /dev/null
+
+BUILD_ARGS=""
+docker_file=""
+CNI_BINARIES_VERSION=$(head -n 1 deps/cni-binaries-version)
+GO_VERSION=$(head -n 1 deps/go-version)
+OVS_VERSION=$(head -n 1 deps/ovs-version-windows)
+
+registry="antrea"
+image_name="antrea-windows"
+image="${registry}/${image_name}"
+BUILD_ARGS="--build-arg GO_VERSION=${GO_VERSION} --build-arg OVS_VERSION=${OVS_VERSION} --build-arg CNI_BINARIES_VERSION=${CNI_BINARIES_VERSION}"
+
+ANTREA_DIR=${THIS_DIR}/../../
+pushd $ANTREA_DIR > /dev/null
+docker_file="build/images/Dockerfile.build.windows"
+docker_build_and_push_windows "${image}" "${docker_file}" "${BUILD_ARGS}" "${BUILD_TAG}" $PUSH "${PULL_OPTION}"
+popd > /dev/null
+
+popd > /dev/null
diff --git a/build/images/deps/nanoserver-version b/build/images/deps/nanoserver-version
deleted file mode 100644
index dfcbacd5d10..00000000000
--- a/build/images/deps/nanoserver-version
+++ /dev/null
@@ -1 +0,0 @@
-1809
diff --git a/build/images/ovs/Dockerfile.windows b/build/images/ovs/Dockerfile.windows
index e5f501ba6b1..e075f720606 100644
--- a/build/images/ovs/Dockerfile.windows
+++ b/build/images/ovs/Dockerfile.windows
@@ -12,42 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG NANOSERVER_VERSION
-
-FROM mcr.microsoft.com/windows/servercore:${NANOSERVER_VERSION} as ovs-base
-
-ARG WIN_OVS_VERSION
-
-# Sets the shell to be PowerShell 
-SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
-
-ENV WIN_OVS_VERSION ${WIN_OVS_VERSION}
-
-# Install VC for ovs
-RUN curl.exe -Lo C:\vc_redist.x64.exe https://aka.ms/vs/17/release/vc_redist.x64.exe; \
-    Start-Process -FilePath C:\vc_redist.x64.exe -ArgumentList ‘/install’,’/quiet’,’/norestart’ -Verb RunAs -Wait; \
-    rm C:\vc_redist.x64.exe;
-
-# Download and extract OVS
-RUN curl.exe -fLo C:\ovs-${env:WIN_OVS_VERSION}-antrea.0-win64.zip https://downloads.antrea.io/ovs/ovs-${env:WIN_OVS_VERSION}-antrea.0-win64.zip; \
-    Expand-Archive C:\ovs-${env:WIN_OVS_VERSION}-antrea.0-win64.zip -DestinationPath C:\; \
-    rm C:\ovs-${env:WIN_OVS_VERSION}-antrea.0-win64.zip;
-
-# Download, extract OpenSSL and copy libraries
-RUN curl.exe -Lo C:\openssl-1.0.2u-x64_86-win64.zip https://indy.fulgan.com/SSL/openssl-1.0.2u-x64_86-win64.zip; \
-    Expand-Archive C:\openssl-1.0.2u-x64_86-win64.zip -DestinationPath C:\openssl; \
-    Remove-Item -Path 'C:\openssl-1.0.2u-x64_86-win64.zip';
-
-FROM mcr.microsoft.com/powershell:lts-nanoserver-${NANOSERVER_VERSION} as windows-ovs
-SHELL ["cmd", "/S", "/C"]
-
-# Switch to a privileged user
-USER ContainerAdministrator
-
-COPY --from=ovs-base /Windows/System32/vcruntime140.dll /Windows/System32/
-COPY --from=ovs-base /openssl/libeay32.dll /Windows/System32/
-COPY --from=ovs-base /openssl/ssleay32.dll /Windows/System32/
-
-COPY --from=ovs-base /openvswitch /openvswitch/
-
-RUN setx /m PATH "C:/openvswitch/usr/bin;C:/openvswitch/usr/sbin;%PATH%"
+FROM --platform=linux/amd64 ubuntu:22.04 as antrea-windows-builder
+ARG OVS_VERSION
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends unzip
+
+ADD https://downloads.antrea.io/ovs/ovs-${OVS_VERSION}-antrea.0-win64.zip ovs-${OVS_VERSION}-antrea.0-win64.zip
+RUN unzip -q ovs-${OVS_VERSION}-antrea.0-win64.zip
+RUN mkdir -p openvswitch/redist
+ADD https://aka.ms/vs/17/release/vc_redist.x64.exe /openvswitch/redist/redist.x64.exe
+ADD https://indy.fulgan.com/SSL/openssl-1.0.2u-x64_86-win64.zip openssl-1.0.2u-x64_86-win64.zip
+RUN mkdir openssl && unzip -q openssl-1.0.2u-x64_86-win64.zip -d openssl && \
+    cp openssl/*.dll /openvswitch/usr/bin/ && \
+    cp openssl/*.dll /openvswitch/usr/sbin/ && \
+    rm -rf openssl openssl-1.0.2u-x64_86-win64.zip
+
+FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0
+COPY --from=antrea-windows-builder /openvswitch /openvswitch
diff --git a/build/images/ovs/README.md b/build/images/ovs/README.md
index 533fc957943..c56c8bc140b 100644
--- a/build/images/ovs/README.md
+++ b/build/images/ovs/README.md
@@ -17,5 +17,12 @@ cd build/images/ovs
 ./build.sh --pull --push
 ```
 
+To build Windows OVS image, you can follow these steps.
+
+```bash
+cd build/images/ovs
+./build.sh --distro windows --push
+```
+
 The script will fail if you do not have permission to push to the `antrea`
 Dockerhub repository.
diff --git a/build/images/ovs/build.sh b/build/images/ovs/build.sh
index 38c95aa2d82..7ac80c43e67 100755
--- a/build/images/ovs/build.sh
+++ b/build/images/ovs/build.sh
@@ -23,12 +23,12 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 source $THIS_DIR/../build-utils.sh
 
-_usage="Usage: $0 [--pull] [--push] [--platform <PLATFORM>] [--distro [ubuntu|ubi]]
+_usage="Usage: $0 [--pull] [--push] [--platform <PLATFORM>] [--distro [ubuntu|ubi|windows]]
 Build the antrea openvswitch image.
         --pull                  Always attempt to pull a newer version of the base images
         --push                  Push the built image to the registry
         --platform <PLATFORM>   Target platform for the image if server is multi-platform capable
-        --distro <distro>       Target Linux distribution
+        --distro <distro>       Target distribution. If distro is 'windows', platform should be empty. The script uses 'windows/amd64' automatically
         --no-cache              Do not use the local build cache nor the cached image from the registry"
 
 PULL=false
@@ -77,7 +77,7 @@ done
 # push the "cache image" to the registry. This functionality is not supported
 # with the default docker driver.
 # See https://docs.docker.com/build/cache/backends/registry/
-if $PUSH && ! check_docker_build_driver "docker-container"; then
+if $PUSH && [ "$DISTRO" != "windows" ] && ! check_docker_build_driver "docker-container"; then
     echoerr "--push requires the docker-container build driver"
     exit 1
 fi
@@ -87,19 +87,24 @@ if [ "$PLATFORM" != "" ] && $PUSH; then
     exit 1
 fi
 
+if [ "$DISTRO" != "ubuntu" ] && [ "$DISTRO" != "ubi" ] && [ "$DISTRO" != "windows" ]; then
+    echoerr "Invalid distribution $DISTRO"
+    exit 1
+fi
+
+OVS_VERSION_FILE="../deps/ovs-version"
+if [ "$DISTRO" == "windows" ]; then
+  OVS_VERSION_FILE="../deps/ovs-version-windows"
+fi
+
 PLATFORM_ARG=""
 if [ "$PLATFORM" != "" ]; then
     PLATFORM_ARG="--platform $PLATFORM"
 fi
 
-if [ "$DISTRO" != "ubuntu" ] && [ "$DISTRO" != "ubi" ]; then
-    echoerr "Invalid distribution $DISTRO"
-    exit 1
-fi
-
 pushd $THIS_DIR > /dev/null
 
-OVS_VERSION=$(head -n 1 ../deps/ovs-version)
+OVS_VERSION=$(head -n 1 ${OVS_VERSION_FILE})
 
 BUILD_TAG=$(../build-tag.sh)
 
@@ -114,6 +119,8 @@ if $PULL; then
     elif [ "$DISTRO" == "ubi" ]; then
         docker pull $PLATFORM_ARG quay.io/centos/centos:stream9
         docker pull $PLATFORM_ARG registry.access.redhat.com/ubi9
+    elif [ "$DISTRO" == "windows" ]; then
+        docker pull --platform linux/amd64 ubuntu:22.04
     fi
 fi
 
@@ -141,6 +148,10 @@ if [ "$DISTRO" == "ubuntu" ]; then
     docker_build_and_push "antrea/openvswitch" "Dockerfile"
 elif [ "$DISTRO" == "ubi" ]; then
     docker_build_and_push "antrea/openvswitch-ubi" "Dockerfile.ubi"
+elif [ "$DISTRO" == "windows" ]; then
+    image="antrea/windows-ovs"
+    build_args="--build-arg OVS_VERSION=$OVS_VERSION"
+    docker_build_and_push_windows "${image}" "Dockerfile.windows" "${build_args}" "${OVS_VERSION}" $PUSH ""
 fi
 
 popd > /dev/null
diff --git a/build/yamls/antrea-windows-with-ovs.yml b/build/yamls/antrea-windows-with-ovs.yml
index ccb7752d351..fe444f28c64 100644
--- a/build/yamls/antrea-windows-with-ovs.yml
+++ b/build/yamls/antrea-windows-with-ovs.yml
@@ -17,6 +17,27 @@ data:
       }
       Write-Host "OVSExt driver has been installed"
     }
+
+    # Check if the VC redistributable is already installed.
+    $OVSRedistDir="$mountPath\openvswitch\redist"
+    if (Test-Path $OVSRedistDir) {
+      $dllFound = $false
+      $paths = $env:PATH -split ';'
+      foreach ($path in $paths) {
+        $dllFiles = Get-ChildItem -Path $path -Filter "vcruntime*.dll" -File -ErrorAction SilentlyContinue
+        if ($dllFiles.Count -gt 0) {
+          $dllFound = $true
+          break
+        }
+      }
+
+      # vcruntime dlls are not installed on the host, then install the binaries.
+      if (-not $dllFound) {
+        Get-ChildItem $OVSRedistDir -Filter *.exe | ForEach-Object {
+          Start-Process -FilePath $_.FullName -Args '/install /passive /norestart' -Verb RunAs -Wait
+        }
+      }
+    }
   Install-WindowsCNI.ps1: |
     $ErrorActionPreference = "Stop";
     mkdir -force c:/var/log/antrea
@@ -303,7 +324,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/agent-windows: 3a8371fd1b923b1176641950074efc943c326c1638293f84ba16bbeec889ce2d
+        checksum/agent-windows: da4b49898e3181dfcb6359f2b2acba452c2ff9a17af8342b6cccb73e8857aad6
         checksum/windows-config: 6ff4f8bd0b310ebe4d4612bdd9697ffb3d79e0e0eab3936420417dd5a8fc128d
         microsoft.com/hostprocess-inherit-user: "true"
       labels:
diff --git a/build/yamls/windows/containerd-with-ovs/conf/Install-OVSDriver.ps1 b/build/yamls/windows/containerd-with-ovs/conf/Install-OVSDriver.ps1
index e49fb1e0121..63dd1089e9b 100644
--- a/build/yamls/windows/containerd-with-ovs/conf/Install-OVSDriver.ps1
+++ b/build/yamls/windows/containerd-with-ovs/conf/Install-OVSDriver.ps1
@@ -14,3 +14,24 @@ if ($driverStatus -like '*not installed*') {
   }
   Write-Host "OVSExt driver has been installed"
 }
+
+# Check if the VC redistributable is already installed.
+$OVSRedistDir="$mountPath\openvswitch\redist"
+if (Test-Path $OVSRedistDir) {
+  $dllFound = $false
+  $paths = $env:PATH -split ';'
+  foreach ($path in $paths) {
+    $dllFiles = Get-ChildItem -Path $path -Filter "vcruntime*.dll" -File -ErrorAction SilentlyContinue
+    if ($dllFiles.Count -gt 0) {
+      $dllFound = $true
+      break
+    }
+  }
+
+  # vcruntime dlls are not installed on the host, then install the binaries.
+  if (-not $dllFound) {
+    Get-ChildItem $OVSRedistDir -Filter *.exe | ForEach-Object {
+      Start-Process -FilePath $_.FullName -Args '/install /passive /norestart' -Verb RunAs -Wait
+    }
+  }
+}
diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
index f2aeda5ec71..8c8c40d3acf 100755
--- a/ci/jenkins/test.sh
+++ b/ci/jenkins/test.sh
@@ -393,13 +393,12 @@ function  build_and_deliver_antrea_windows_and_linux_images {
 
     cp -f build/yamls/*.yml $WORKDIR
     set +e
-    deliver_antrea_windows &> deliver_antrea_windows.log &
-    deliver_antrea_windows=$!
+    deliver_antrea_windows
+    windows_result=$?
     deliver_antrea_linux
     linux_result=$?
     wait $deliver_antrea_linux
-    windows_result=$?
-    cat deliver_antrea_windows.log
+    wait $deliver_antrea_windows
     if [ $windows_result -ne 0 ] || [ $linux_result -ne 0 ]; then
         exit 1
     fi
@@ -455,21 +454,14 @@ function deliver_antrea_linux {
 }
 
 function deliver_antrea_windows {
-    echo "===== Build Antrea Windows on Windows Jumper Node ====="
-    echo "==== Reverting Windows VM ${WIN_IMAGE_NODE} ====="
-    revert_snapshot_windows ${WIN_IMAGE_NODE}
+    echo "===== Build Antrea Windows ====="
     rm -f antrea-windows.tar.gz
-    # Compress antrea repo and copy it to a Windows node
-    mkdir -p jenkins
-    tar --exclude='./jenkins' -czf jenkins/antrea_repo.tar.gz -C "$(pwd)" .
-    for i in `seq 2`; do
-        timeout 2m scp -o StrictHostKeyChecking=no -T jenkins/antrea_repo.tar.gz Administrator@${IP}: && break
-    done
-    ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver && docker tag ${DOCKER_REGISTRY}/antrea/golang:${GO_VERSION}-nanoserver golang:${GO_VERSION}-nanoserver"
-    ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "rm -rf antrea && mkdir antrea && cd antrea && tar -xzf ../antrea_repo.tar.gz > /dev/null && NO_PULL=${NO_PULL}; DOCKER_NETWORK=host make build-windows && docker save -o antrea-windows.tar antrea/antrea-windows:latest && gzip -f antrea-windows.tar" || true
-    for i in `seq 2`; do
-        timeout 2m scp -o StrictHostKeyChecking=no -T Administrator@${IP}:antrea/antrea-windows.tar.gz . && break
-    done
+    make build-windows
+    if ! (test -f antrea-windows.tar); then
+        echo "antrea-windows.tar wasn't built, exiting"
+        exit 1
+    fi
+    gzip -f antrea-windows.tar
 
     echo "===== Deliver Antrea Windows to Windows worker nodes and pull necessary images on Windows worker nodes ====="
     sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1
@@ -488,22 +480,17 @@ function deliver_antrea_windows {
         for i in "${!k8s_images[@]}"; do
             ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull ${k8s_images[i]} && ctr -n k8s.io images tag ${k8s_images[i]} ${e2e_images[i]}" || true
         done
-        if ! (test -f antrea-windows.tar.gz); then
-            echo "Windows VM ${WIN_IMAGE_NODE} didn't build antrea-windows.tar.gz, exiting"
-            exit 1
-        else
-            for i in `seq 2`; do
-                timeout 2m scp -o StrictHostKeyChecking=no -T antrea-windows.tar.gz Administrator@${IP}: && break
-            done
-            ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "gzip -d antrea-windows.tar.gz && ctr -n k8s.io images import antrea-windows.tar"
-        fi
+
+        for i in `seq 2`; do
+            timeout 2m scp -o StrictHostKeyChecking=no -T antrea-windows.tar.gz Administrator@${IP}: && break
+        done
+        ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "gzip -d antrea-windows.tar.gz && ctr -n k8s.io images import antrea-windows.tar"
     done
     # Add Windows interface DHCP check using CI script to obtain the original interface status.
     WINIP=$(kubectl get nodes -o wide --no-headers=true | awk '$1 ~ /win-0/ {print $6}')
     WIN_DHCP=$(ssh -o StrictHostKeyChecking=no -n administrator@${WINIP} 'powershell.exe "(Get-NetIPInterface -InterfaceAlias \"Ethernet0 2\" -AddressFamily IPv4).Dhcp"')
     echo "Original adapter DHCP status: $WIN_DHCP"
     echo $WIN_DHCP > WIN_DHCP
-    rm -f antrea-windows.tar
     echo "==== Finish building and delivering Windows images ===="
 }
 
diff --git a/docs/windows.md b/docs/windows.md
index a6e0b14b8a4..810903d2c66 100644
--- a/docs/windows.md
+++ b/docs/windows.md
@@ -79,6 +79,10 @@ prefer running the Antrea Agent as a Windows service, or if you are using
 containerd 1.5, you can use the [Installation as a
 Service](#installation-as-a-service) method.
 
+Starting from v2.1, Antrea Windows image is built on Linux host with docker buildx
+and uses [hpc](https://github.com/microsoft/windows-host-process-containers-base-image)
+as the base image.
+
 ### Prerequisites
 
 * Create a Kubernetes cluster.
diff --git a/hack/build-antrea-windows-all.sh b/hack/build-antrea-windows-all.sh
index 9e67f84c459..064639d9ced 100755
--- a/hack/build-antrea-windows-all.sh
+++ b/hack/build-antrea-windows-all.sh
@@ -23,20 +23,23 @@ function echoerr {
     >&2 echo "$@"
 }
 
-_usage="Usage: $0 [--pull] [--push-base-images]
+_usage="Usage: $0 [--pull] [--push] [--push-base-images]
 Build the antrea/antrea-windows image, as well as all the base images in the build chain. This is
 typically used in CI to build the image with the latest version of all dependencies, taking into
 account changes to all Dockerfiles.
         --pull                  Always attempt to pull a newer version of the base images.
-        --push-base-images      Push built images to the registry. Only base images will be pushed.
-This script must run on a Windows machine!"
+        --push                  Push built antrea/antrea-windows image to the registry.
+        --push-base-images      Push built images to the registry. Only Windows OVS image will be pushed.
+
+This script is run on a Linux machine."
 
 function print_usage {
     echoerr "$_usage"
 }
 
 PULL=false
-PUSH=false
+PUSH_BASE=false
+PUSH_AGENT=false
 
 while [[ $# -gt 0 ]]
 do
@@ -48,7 +51,11 @@ case $key in
     shift
     ;;
     --push-base-images)
-    PUSH=true
+    PUSH_BASE=true
+    shift
+    ;;
+    --push)
+    PUSH_AGENT=true
     shift
     ;;
     -h|--help)
@@ -66,65 +73,23 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 pushd "$THIS_DIR/.." > /dev/null
 
-NANOSERVER_VERSION=$(head -n 1 build/images/deps/nanoserver-version)
-CNI_BINARIES_VERSION=$(head -n 1 build/images/deps/cni-binaries-version)
-GO_VERSION=$(head -n 1 build/images/deps/go-version)
-WIN_OVS_VERSION=$(head -n 1 build/images/deps/ovs-version-windows)
-WIN_BUILD_OVS_TAG=$(echo $NANOSERVER_VERSION-$WIN_OVS_VERSION)
-WIN_BUILD_TAG=$(echo $GO_VERSION $CNI_BINARIES_VERSION $NANOSERVER_VERSION| md5sum| head -c 10)
-
-echo "WIN_BUILD_TAG=$WIN_BUILD_TAG"
-
+ARGS=""
 if $PULL; then
-    docker pull mcr.microsoft.com/windows/servercore:$NANOSERVER_VERSION
-    docker pull golang:$GO_VERSION-nanoserver
-    docker pull mcr.microsoft.com/windows/nanoserver:$NANOSERVER_VERSION
-    docker pull mcr.microsoft.com/powershell:lts-nanoserver-$NANOSERVER_VERSION
-    docker pull antrea/windows-utility-base:$WIN_BUILD_TAG || true
-    docker pull antrea/windows-golang:$WIN_BUILD_TAG || true
-    docker pull antrea/base-windows:$WIN_BUILD_TAG || true
-    docker pull antrea/windows-ovs:$WIN_BUILD_OVS_TAG || true
+    ARGS="$ARGS --pull"
 fi
 
-cd build/images/base-windows
-docker build --target windows-utility-base \
-       --cache-from antrea/windows-utility-base:$WIN_BUILD_TAG \
-       -t antrea/windows-utility-base:$WIN_BUILD_TAG \
-       --build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION \
-       --build-arg NANOSERVER_VERSION=$NANOSERVER_VERSION .
-docker build --target windows-golang \
-       --cache-from antrea/windows-golang:$WIN_BUILD_TAG \
-       -t antrea/windows-golang:$WIN_BUILD_TAG \
-       --build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION \
-       --build-arg GO_VERSION=$GO_VERSION \
-       --build-arg NANOSERVER_VERSION=$NANOSERVER_VERSION .
-docker build \
-       --cache-from antrea/windows-utility-base:$WIN_BUILD_TAG \
-       --cache-from antrea/windows-golang:$WIN_BUILD_TAG \
-       --cache-from antrea/base-windows:$WIN_BUILD_TAG \
-       -t antrea/base-windows:$WIN_BUILD_TAG \
-       --build-arg CNI_BINARIES_VERSION=$CNI_BINARIES_VERSION \
-       --build-arg GO_VERSION=$GO_VERSION \
-       --build-arg NANOSERVER_VERSION=$NANOSERVER_VERSION .
-cd -
+if $PUSH_BASE; then
+   ARGS="$ARGS --push"
+fi
 
 cd build/images/ovs
-
-docker build --target windows-ovs -f Dockerfile.windows \
-        -t antrea/windows-ovs:$WIN_BUILD_OVS_TAG \
-        --build-arg WIN_OVS_VERSION=$WIN_OVS_VERSION \
-        --build-arg NANOSERVER_VERSION=$NANOSERVER_VERSION .
+./build.sh --distro windows $ARGS
 cd -
 
-if $PUSH; then
-    docker push antrea/windows-utility-base:$WIN_BUILD_TAG
-    docker push antrea/windows-golang:$WIN_BUILD_TAG
-    docker push antrea/base-windows:$WIN_BUILD_TAG
-    docker push antrea/windows-ovs:$WIN_BUILD_OVS_TAG
+if $PUSH_AGENT; then
+    make build-and-push-windows
+else
+    make build-windows
 fi
 
-export NO_PULL=1
-
-make build-windows
-
 popd > /dev/null