diff --git a/CMakeLists.txt b/CMakeLists.txt
index 17a8e11e068..b9538ae5324 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,7 +8,14 @@ project(onnx-mlir)
 option(ONNX_MLIR_BUILD_TESTS "Build ONNX-MLIR test executables. If OFF, just generate build targets." ON)
 option(ONNX_MLIR_SUPPRESS_THIRD_PARTY_WARNINGS "Suppress warning in third_party code." ON)
 
+# On systems that still have legacy lib64 directories (e.g., rhel/fedora,
+# etc.), by default some components (e.g., cmake) install into lib while
+# others (e.g., python) install into lib64.
+# This causes trouble when we try to figure out the runtime directory in
+# CompilerUtils.cpp::getRuntimeDir(). So we explicitly set CMAKE_INSTALL_LIBDIR
+# to install into lib on all systems so we don't have to deal with lib64.
 set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_INSTALL_LIBDIR lib)
 
 if (NOT CMAKE_BUILD_TYPE)
   message(STATUS "No build type selected, default to Debug")
diff --git a/src/Compiler/CompilerUtils.cpp b/src/Compiler/CompilerUtils.cpp
index 6fc9f5a3336..e4ddb1da731 100644
--- a/src/Compiler/CompilerUtils.cpp
+++ b/src/Compiler/CompilerUtils.cpp
@@ -163,12 +163,15 @@ static std::string getExecPath() {
 //
 //   - if ONNX_MLIR_RUNTIME_DIR is set, use it, otherwise
 //   - get path from where onnx-mlir is run, if it's of the form
-//   /foo/bar/bin/onnx-mlir,
+//     /foo/bar/bin/onnx-mlir,
 //     the runtime directory is /foo/bar/lib (note that when onnx-mlir is
 //     installed system wide, which is typically /usr/local/bin, this will
 //     correctly resolve to /usr/local/lib), but some systems still have
 //     lib64 so we check that first. If neither exists, then
 //   - use CMAKE_INSTALL_PREFIX/lib, which is typically /usr/local/lib
+//
+// We now explicitly set CMAKE_INSTALL_LIBDIR to lib so we don't have
+// to deal with lib64 anymore.
 static std::string getRuntimeDir() {
   const auto &envDir = getEnvVar("ONNX_MLIR_RUNTIME_DIR");
   if (envDir && llvm::sys::fs::exists(envDir.getValue()))
@@ -177,18 +180,10 @@ static std::string getRuntimeDir() {
   string execDir = llvm::sys::path::parent_path(getExecPath()).str();
   if (llvm::sys::path::stem(execDir).str().compare("bin") == 0) {
     string p = execDir.substr(0, execDir.size() - 3);
-    if (llvm::sys::fs::exists(p + "lib64"))
-      return p + "lib64";
     if (llvm::sys::fs::exists(p + "lib"))
       return p + "lib";
   }
 
-  llvm::SmallString<8> instDir64(kInstPath);
-  llvm::sys::path::append(instDir64, "lib64");
-  string p = llvm::StringRef(instDir64).str();
-  if (llvm::sys::fs::exists(p))
-    return p;
-
   llvm::SmallString<8> instDir(kInstPath);
   llvm::sys::path::append(instDir, "lib");
   return llvm::StringRef(instDir).str();
@@ -200,7 +195,7 @@ static std::string getRuntimeDir() {
 // and its source has been removed.
 //
 // To account for this scenario, we first search for the tools in the same
-// directory where onnx-mlir is run. If they are found, it  means both onnx-mlir
+// directory where onnx-mlir is run. If they are found, it means both onnx-mlir
 // and llvm-project have been installed system wide under the same directory,
 // so we get them from that directory (typically /usr/local/bin). Otherwise,
 // at least one of onnx-mlir and llvm-project has not been installed system