diff --git a/cmake/onnxruntime-linux-aarch64-gpu.cmake b/cmake/onnxruntime-linux-aarch64-gpu.cmake
new file mode 100644
index 000000000..83ca7560f
--- /dev/null
+++ b/cmake/onnxruntime-linux-aarch64-gpu.cmake
@@ -0,0 +1,84 @@
+# Copyright (c)  2022-2023  Xiaomi Corporation
+message(STATUS "CMAKE_SYSTEM_NAME: ${CMAKE_SYSTEM_NAME}")
+message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
+
+if(NOT CMAKE_SYSTEM_NAME STREQUAL Linux)
+  message(FATAL_ERROR "This file is for Linux only. Given: ${CMAKE_SYSTEM_NAME}")
+endif()
+
+if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
+  message(FATAL_ERROR "This file is for aarch64 only. Given: ${CMAKE_SYSTEM_PROCESSOR}")
+endif()
+
+if(NOT SHERPA_ONNX_ENABLE_GPU)
+  message(FATAL_ERROR "This file is for NVIDIA GPU only. Given SHERPA_ONNX_ENABLE_GPU: ${SHERPA_ONNX_ENABLE_GPU}")
+endif()
+
+set(onnxruntime_URL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v1.16.0/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2")
+set(onnxruntime_URL2 )
+set(onnxruntime_HASH "SHA256=c1fd7c9b2f38f5ca9ca8950bd4c34e175c4d0a7eeaeef666474838d9f5902d00")
+
+# If you don't have access to the Internet,
+# please download onnxruntime to one of the following locations.
+# You can add more if you want.
+set(possible_file_locations
+  $ENV{HOME}/Downloads/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2
+  ${PROJECT_SOURCE_DIR}/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2
+  ${PROJECT_BINARY_DIR}/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2
+  /tmp/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2
+  /star-fj/fangjun/download/github/onnxruntime-gpu-1.15.1-linux-aarch64.tar.bz2
+)
+
+foreach(f IN LISTS possible_file_locations)
+  if(EXISTS ${f})
+    set(onnxruntime_URL  "${f}")
+    file(TO_CMAKE_PATH "${onnxruntime_URL}" onnxruntime_URL)
+    message(STATUS "Found local downloaded onnxruntime: ${onnxruntime_URL}")
+    set(onnxruntime_URL2)
+    break()
+  endif()
+endforeach()
+
+FetchContent_Declare(onnxruntime
+  URL
+    ${onnxruntime_URL}
+    ${onnxruntime_URL2}
+  URL_HASH          ${onnxruntime_HASH}
+)
+
+FetchContent_GetProperties(onnxruntime)
+if(NOT onnxruntime_POPULATED)
+  message(STATUS "Downloading onnxruntime from ${onnxruntime_URL}")
+  FetchContent_Populate(onnxruntime)
+endif()
+message(STATUS "onnxruntime is downloaded to ${onnxruntime_SOURCE_DIR}")
+
+find_library(location_onnxruntime_cuda_lib onnxruntime_providers_cuda
+  PATHS
+  "${onnxruntime_SOURCE_DIR}/capi"
+  NO_CMAKE_SYSTEM_PATH
+)
+
+add_library(onnxruntime_providers_cuda SHARED IMPORTED)
+set_target_properties(onnxruntime_providers_cuda PROPERTIES
+  IMPORTED_LOCATION ${location_onnxruntime_cuda_lib}
+  INTERFACE_INCLUDE_DIRECTORIES "${onnxruntime_SOURCE_DIR}/include"
+)
+message(STATUS "location_onnxruntime_cuda_lib: ${location_onnxruntime_cuda_lib}")
+
+# for libonnxruntime_providers_shared.so
+find_library(location_onnxruntime_providers_shared_lib onnxruntime_providers_shared
+  PATHS
+  "${onnxruntime_SOURCE_DIR}/capi"
+  NO_CMAKE_SYSTEM_PATH
+)
+add_library(onnxruntime_providers_shared SHARED IMPORTED)
+set_target_properties(onnxruntime_providers_shared PROPERTIES
+  IMPORTED_LOCATION ${location_onnxruntime_providers_shared_lib}
+  INTERFACE_INCLUDE_DIRECTORIES "${onnxruntime_SOURCE_DIR}/include"
+)
+message(STATUS "location_onnxruntime_providers_shared_lib: ${location_onnxruntime_providers_shared_lib}")
+
+file(GLOB onnxruntime_lib_files "${onnxruntime_SOURCE_DIR}/capi/libonnxruntime*")
+message(STATUS "onnxruntime lib files: ${onnxruntime_lib_files}")
+install(FILES ${onnxruntime_lib_files} DESTINATION lib)
diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake
index 75e147ca1..6b6bb20e4 100644
--- a/cmake/onnxruntime.cmake
+++ b/cmake/onnxruntime.cmake
@@ -6,7 +6,11 @@ function(download_onnxruntime)
   message(STATUS "CMAKE_SYSTEM_PROCESSOR: ${CMAKE_SYSTEM_PROCESSOR}")
 
   if(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64)
-    include(onnxruntime-linux-aarch64)
+    if(SHERPA_ONNX_ENABLE_GPU)
+      include(onnxruntime-linux-aarch64-gpu)
+    else()
+      include(onnxruntime-linux-aarch64)
+    endif()
   elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND (CMAKE_SYSTEM_PROCESSOR STREQUAL arm OR CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l))
     include(onnxruntime-linux-arm)
   elseif(CMAKE_SYSTEM_NAME STREQUAL Linux AND CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt
index 386b81ba3..5d50748ce 100644
--- a/sherpa-onnx/csrc/CMakeLists.txt
+++ b/sherpa-onnx/csrc/CMakeLists.txt
@@ -98,7 +98,9 @@ endif()
 target_link_libraries(sherpa-onnx-core kaldi-native-fbank-core)
 
 if(BUILD_SHARED_LIBS OR APPLE OR CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm)
-  target_link_libraries(sherpa-onnx-core onnxruntime)
+  if (TARGET onnxruntime)
+    target_link_libraries(sherpa-onnx-core onnxruntime)
+  endif()
 else()
   target_link_libraries(sherpa-onnx-core ${onnxruntime_lib_files})
 endif()