diff --git a/Makefile b/Makefile
index 096bd1fbbd7..a05e568471e 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,10 @@ SHELL              := /bin/bash
 GO                 ?= go
 LDFLAGS            :=
 GOFLAGS            :=
+# By default, disable cgo for all Go binaries.
+# For binaries meant to be published as release assets or copied to a different host, cgo should
+# always be disabled.
+CGO_ENABLED        ?= 0
 BINDIR             ?= $(CURDIR)/bin
 GO_FILES           := $(shell find . -type d -name '.cache' -prune -o -type f -name '*.go' -print)
 GOPATH             ?= $$($(GO) env GOPATH)
@@ -33,6 +37,8 @@ WIN_BUILD_ARGS += --build-arg NANOSERVER_VERSION=$(NANOSERVER_VERSION)
 WIN_BUILD_ARGS += --build-arg WIN_BUILD_TAG=$(WIN_BUILD_TAG)
 WIN_BUILD_ARGS += --build-arg WIN_BUILD_OVS_TAG=$(WIN_BUILD_OVS_TAG)
 
+export CGO_ENABLED
+
 .PHONY: all
 all: build
 
@@ -78,7 +84,7 @@ antrea-agent:
 .PHONY: antrea-agent-release
 antrea-agent-release:
 	@mkdir -p $(BINDIR)
-	@CGO_ENABLED=0 $(GO) build -o $(BINDIR)/$(ANTREA_AGENT_BINARY_NAME) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-agent
+	$(GO) build -o $(BINDIR)/$(ANTREA_AGENT_BINARY_NAME) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-agent
 
 .PHONY: antrea-agent-simulator
 antrea-agent-simulator:
@@ -104,29 +110,25 @@ antrea-controller-instr-binary:
 	@mkdir -p $(BINDIR)
 	GOOS=linux $(GO) test -tags testbincover -covermode count -coverpkg=antrea.io/antrea/pkg/... -c -o $(BINDIR)/antrea-controller-coverage $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-controller
 
-# diable cgo for antrea-cni since it can be installed on some systems with
-# incompatible or missing system libraries.
 .PHONY: antrea-cni
 antrea-cni:
 	@mkdir -p $(BINDIR)
-	GOOS=linux CGO_ENABLED=0 $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni
 
 .PHONY: antrea-cni
 antrea-cni-release:
 	@mkdir -p $(BINDIR)
-	@CGO_ENABLED=0 $(GO) build -o $(BINDIR)/$(ANTREA_CNI_BINARY_NAME) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni
+	$(GO) build -o $(BINDIR)/$(ANTREA_CNI_BINARY_NAME) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni
 
 .PHONY: antctl-instr-binary
 antctl-instr-binary:
 	@mkdir -p $(BINDIR)
 	GOOS=linux $(GO) test -tags testbincover -covermode count -coverpkg=antrea.io/antrea/pkg/... -c -o $(BINDIR)/antctl-coverage $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antctl
 
-# diable cgo for antrea-cni and antrea-agent: antrea-cni is meant to be
-# installed on the host and the antrea-agent is run as a process on Windows.
 .PHONY: windows-bin
 windows-bin:
 	@mkdir -p $(BINDIR)
-	GOOS=windows CGO_ENABLED=0 $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni antrea.io/antrea/cmd/antrea-agent antrea.io/antrea/cmd/antctl
+	GOOS=windows $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/cmd/antrea-cni antrea.io/antrea/cmd/antrea-agent antrea.io/antrea/cmd/antctl
 
 .PHONY: flow-aggregator
 flow-aggregator:
@@ -234,7 +236,7 @@ antctl: $(ANTCTL_BINARIES)
 
 .PHONY: antctl-release
 antctl-release:
-	@CGO_ENABLED=0 $(GO) build -o $(BINDIR)/$(ANTCTL_BINARY_NAME) $(GOFLAGS) -ldflags '-s -w $(LDFLAGS)' antrea.io/antrea/cmd/antctl
+	$(GO) build -o $(BINDIR)/$(ANTCTL_BINARY_NAME) $(GOFLAGS) -ldflags '-s -w $(LDFLAGS)' antrea.io/antrea/cmd/antctl
 
 .PHONY: check-copyright
 check-copyright: 
@@ -244,11 +246,13 @@ check-copyright:
 add-copyright: 
 	@GO=$(GO) $(CURDIR)/hack/add-license.sh --add
 
+# Cgo is required to run the race detector.
+
 .PHONY: .linux-test-unit
 .linux-test-unit: .coverage
 	@echo
 	@echo "==> Running unit tests <=="
-	$(GO) test -race -coverpkg=antrea.io/antrea/cmd/...,antrea.io/antrea/pkg/...,antrea.io/antrea/multicluster/cmd/...,antrea.io/antrea/multicluster/controllers/... \
+	CGO_ENABLED=1 $(GO) test -race -coverpkg=antrea.io/antrea/cmd/...,antrea.io/antrea/pkg/...,antrea.io/antrea/multicluster/cmd/...,antrea.io/antrea/multicluster/controllers/... \
 	  -coverprofile=.coverage/coverage-unit.txt -covermode=atomic \
 	  antrea.io/antrea/cmd/... antrea.io/antrea/pkg/... antrea.io/antrea/multicluster/cmd/... antrea.io/antrea/multicluster/controllers/...
 
@@ -256,7 +260,7 @@ add-copyright:
 .windows-test-unit: .coverage
 	@echo
 	@echo "==> Running unit tests <=="
-	$(GO) test -race -coverpkg=antrea.io/antrea/cmd/...,antrea.io/antrea/pkg/... \
+	CGO_ENABLED=1 $(GO) test -race -coverpkg=antrea.io/antrea/cmd/...,antrea.io/antrea/pkg/... \
 	  -coverprofile=.coverage/coverage-unit.txt -covermode=atomic \
 	  antrea.io/antrea/cmd/... antrea.io/antrea/pkg/...
 
diff --git a/build/images/Dockerfile.build.agent.coverage b/build/images/Dockerfile.build.agent.coverage
index 1e2e36a3e8f..e318b41f2a6 100644
--- a/build/images/Dockerfile.build.agent.coverage
+++ b/build/images/Dockerfile.build.agent.coverage
@@ -24,12 +24,9 @@ RUN go mod download
 
 COPY . /antrea
 
+RUN make antctl-linux antctl-instr-binary && mv bin/antctl-linux bin/antctl
+
 RUN make antrea-agent antrea-cni antrea-agent-instr-binary
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
-RUN CGO_ENABLED=0 make antctl-linux antctl-instr-binary
-RUN mv bin/antctl-linux bin/antctl
 
 FROM antrea/base-ubuntu:${BUILD_TAG}
 
diff --git a/build/images/Dockerfile.build.agent.ubi b/build/images/Dockerfile.build.agent.ubi
index 5eef966a658..634212d384f 100644
--- a/build/images/Dockerfile.build.agent.ubi
+++ b/build/images/Dockerfile.build.agent.ubi
@@ -48,13 +48,11 @@ COPY . /antrea
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-agent antrea-cni
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-agent antrea-cni
 
 FROM antrea/base-ubi:${BUILD_TAG}
 
diff --git a/build/images/Dockerfile.build.agent.ubuntu b/build/images/Dockerfile.build.agent.ubuntu
index 14636d8e467..4d1168aff46 100644
--- a/build/images/Dockerfile.build.agent.ubuntu
+++ b/build/images/Dockerfile.build.agent.ubuntu
@@ -27,13 +27,11 @@ COPY . /antrea
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-agent antrea-cni
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-agent antrea-cni
 
 FROM antrea/base-ubuntu:${BUILD_TAG}
 
diff --git a/build/images/Dockerfile.build.controller.coverage b/build/images/Dockerfile.build.controller.coverage
index 3675a88e71d..6e3499f76d9 100644
--- a/build/images/Dockerfile.build.controller.coverage
+++ b/build/images/Dockerfile.build.controller.coverage
@@ -24,12 +24,9 @@ RUN go mod download
 
 COPY . /antrea
 
+RUN make antctl-linux antctl-instr-binary && mv bin/antctl-linux bin/antctl
+
 RUN make antrea-controller antrea-controller-instr-binary
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
-RUN CGO_ENABLED=0 make antctl-linux antctl-instr-binary
-RUN mv bin/antctl-linux bin/antctl
 
 FROM ubuntu:22.04
 
diff --git a/build/images/Dockerfile.build.controller.ubi b/build/images/Dockerfile.build.controller.ubi
index c674e5e61d0..5bb43b96c7b 100644
--- a/build/images/Dockerfile.build.controller.ubi
+++ b/build/images/Dockerfile.build.controller.ubi
@@ -48,13 +48,11 @@ COPY . /antrea
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-controller
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-controller
 
 FROM ubuntu:22.04
 
diff --git a/build/images/Dockerfile.build.controller.ubuntu b/build/images/Dockerfile.build.controller.ubuntu
index e8908b591d2..9a2e03bfb21 100644
--- a/build/images/Dockerfile.build.controller.ubuntu
+++ b/build/images/Dockerfile.build.controller.ubuntu
@@ -27,13 +27,11 @@ COPY . /antrea
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-controller
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-controller
 
 FROM ubuntu:22.04
 
diff --git a/build/images/Dockerfile.build.coverage b/build/images/Dockerfile.build.coverage
index 8065158ac31..d54245db2c5 100644
--- a/build/images/Dockerfile.build.coverage
+++ b/build/images/Dockerfile.build.coverage
@@ -24,12 +24,15 @@ RUN go mod download
 
 COPY . /antrea
 
-RUN make antrea-agent antrea-controller antrea-cni antrea-controller-instr-binary antrea-agent-instr-binary
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
-RUN CGO_ENABLED=0 make antctl-linux antctl-instr-binary
-RUN mv bin/antctl-linux bin/antctl
+# Build antctl first in order to share an extra layer with
+# build/images/Dockerfile.build.agent.coverage and build/images/Dockerfile.build.controller.coverage.
+RUN make antctl-linux antctl-instr-binary && mv bin/antctl-linux bin/antctl
+
+# Then build antrea-agent and antrea-cni, in order to share an extra layer with
+# build/images/Dockerfile.build.agent.coverage.
+RUN make antrea-agent antrea-cni antrea-agent-instr-binary
+
+RUN make antrea-controller antrea-controller-instr-binary
 
 FROM antrea/base-ubuntu:${BUILD_TAG}
 
diff --git a/build/images/Dockerfile.build.ubi b/build/images/Dockerfile.build.ubi
index eb9e97e6f87..5f5f9488e9c 100644
--- a/build/images/Dockerfile.build.ubi
+++ b/build/images/Dockerfile.build.ubi
@@ -12,30 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG BUILD_TAG
-FROM registry.access.redhat.com/ubi8 as antrea-build
-
-ADD https://go.dev/dl/?mode=json&include=all go-versions.json
-
-RUN yum install ca-certificates gcc git jq make wget -y
-
 ARG GO_VERSION
-
-# GO_VERSION is a Go minor version, we use the downloaded go-versions.json file
-# to identify and install the latest patch release for this minor version.
-RUN set -eux; \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-         x86_64) goArch='amd64' ;; \
-         arm) goArch='armv6l' ;; \
-         aarch64) goArch='arm64' ;; \
-         *) goArch=''; echo >&2; echo >&2 "unsupported architecture '$arch'"; echo >&2 ; exit 1 ;; \
-    esac; \
-    GO_ARCHIVE=$(jq --arg version_prefix "go${GO_VERSION}." --arg arch "$goArch" -r '. | map(select(. | .version | startswith($version_prefix))) | first | .files[] | select(.os == "linux" and .arch == $arch and .kind == "archive").filename' go-versions.json); \
-    wget -q -O - https://go.dev/dl/${GO_ARCHIVE} | tar xz -C /usr/local/
-
-# Using ENV makes the change persistent, but this is just a builder image.
-ENV PATH /usr/local/go/bin:$PATH
+ARG BUILD_TAG
+FROM golang:${GO_VERSION} as antrea-build
 
 WORKDIR /antrea
 
@@ -46,15 +25,21 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \
 
 COPY . /antrea
 
+# Build antctl first in order to share an extra layer with
+# build/images/Dockerfile.build.agent.ubi and build/images/Dockerfile.build.controller.ubi.
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-agent antrea-controller antrea-cni
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
+# Then build antrea-agent and antrea-cni, in order to share an extra layer with
+# build/images/Dockerfile.build.agent.ubi.
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-cni
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-controller
 
 FROM antrea/base-ubi:${BUILD_TAG}
 
diff --git a/build/images/Dockerfile.build.ubuntu b/build/images/Dockerfile.build.ubuntu
index a07654f6ae9..0e2c3e8dfd4 100644
--- a/build/images/Dockerfile.build.ubuntu
+++ b/build/images/Dockerfile.build.ubuntu
@@ -25,15 +25,21 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \
 
 COPY . /antrea
 
+# Build antctl first in order to share an extra layer with
+# build/images/Dockerfile.build.agent.ubuntu and build/images/Dockerfile.build.controller.ubuntu.
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    make antrea-agent antrea-controller antrea-cni
-# Disable CGO for antctl in case it is copied outside of the container image. It
-# also reduces the size of the binary and aligns with how we distribute antctl
-# in release assets.
+    make antctl-linux && mv bin/antctl-linux bin/antctl
+
+# Then build antrea-agent and antrea-cni, in order to share an extra layer with
+# build/images/Dockerfile.build.agent.ubuntu.
+RUN --mount=type=cache,target=/go/pkg/mod/ \
+    --mount=type=cache,target=/root/.cache/go-build/ \
+    make antrea-agent antrea-cni
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build/ \
-    CGO_ENABLED=0 make antctl-linux && mv bin/antctl-linux bin/antctl
+    make antrea-controller
 
 FROM antrea/base-ubuntu:${BUILD_TAG}
 
diff --git a/hack/release/prepare-assets.sh b/hack/release/prepare-assets.sh
index 4416ad32dd8..66e1fa42faf 100755
--- a/hack/release/prepare-assets.sh
+++ b/hack/release/prepare-assets.sh
@@ -49,6 +49,9 @@ pushd $THIS_DIR/../.. > /dev/null
 mkdir -p "$1"
 OUTPUT_DIR=$(cd "$1" && pwd)
 
+# Cgo should always be disabled for release assets.
+export CGO_ENABLED=0
+
 ANTREA_BUILDS=(
     "linux amd64 linux-x86_64"
     "linux arm64 linux-arm64"
@@ -63,8 +66,6 @@ for build in "${ANTREA_BUILDS[@]}"; do
     arch="${args[1]}"
     suffix="${args[2]}"
 
-    # all "*-release" targets disable cgo, which is appropriate when
-    # distributing release assets, for portability.
     GOOS=$os GOARCH=$arch ANTCTL_BINARY_NAME="antctl-$suffix" BINDIR="$OUTPUT_DIR" make antctl-release
 done