Improve inference performance

quic · Dec 4, 2024 · 216166c · 216166c
1 parent cf53f1f
commit 216166c
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 13 deletions.
diff --git a/src/LibAppBuilder.cpp b/src/LibAppBuilder.cpp
@@ -377,6 +377,12 @@ bool ModelInitializeEx(const std::string& model_name, const std::string& proc_na
       }
     }
 
+    // improve performance.
+    if (sample_app::StatusCode::SUCCESS != app->setupInputAndOutputTensors()) {
+      app->reportError("Setup Input and Output Tensors failure");
+      return false;
+    }
+
     if (loadFromCachedBinary) {
         if (sample_app::StatusCode::SUCCESS != app->initializePerformance()) {
             app->reportError("Performance initialization failure");
@@ -452,6 +458,12 @@ bool ModelDestroyEx(std::string model_name, std::string proc_name) {
         return false;
     }
 
+    // improve performance.
+    if (sample_app::StatusCode::SUCCESS != app->tearDownInputAndOutputTensors()) {
+        app->reportError("Input and Output Tensors destroy failure");
+        return false;
+    }
+
     if (sample_app::StatusCode::SUCCESS != app->destroyPerformance()) {
         app->reportError("Performance destroy failure");
         return false;

diff --git a/src/QnnSampleApp.cpp b/src/QnnSampleApp.cpp
@@ -843,6 +843,46 @@ void bufferToFile(std::vector<uint8_t*>& buffers, std::vector<size_t>& size, std
 }
 #endif
 
+// improve performance.
+sample_app::StatusCode sample_app::QnnSampleApp::setupInputAndOutputTensors()
+{
+  auto returnStatus = qnn::tools::iotensor::StatusCode::SUCCESS;
+
+  for (size_t graphIdx = 0; graphIdx < m_graphsCount; graphIdx++) {
+    auto& graphInfo = (*m_graphsInfo)[graphIdx];
+    Qnn_Tensor_t** inputs  = &(graphInfo.m_inputs );
+    Qnn_Tensor_t** outputs = &(graphInfo.m_outputs);
+    returnStatus = m_ioTensor.setupInputAndOutputTensors(inputs, outputs, graphInfo);
+    if (qnn::tools::iotensor::StatusCode::SUCCESS != returnStatus) {
+      QNN_ERROR("Error in setting up Input and output Tensors for graphIdx: %d", graphIdx);
+      break;
+    }
+  }
+
+  return static_cast<sample_app::StatusCode>(returnStatus);
+}
+
+// improve performance.
+sample_app::StatusCode sample_app::QnnSampleApp::tearDownInputAndOutputTensors()
+{
+  auto returnStatus = qnn::tools::iotensor::StatusCode::SUCCESS;
+
+  for (size_t graphIdx = 0; graphIdx < m_graphsCount; graphIdx++) {
+    auto& graphInfo = (*m_graphsInfo)[graphIdx];
+    Qnn_Tensor_t* inputs  = graphInfo.m_inputs ;
+    Qnn_Tensor_t* outputs = graphInfo.m_outputs;
+    returnStatus = m_ioTensor.tearDownInputAndOutputTensors(inputs, outputs, graphInfo.numInputTensors, graphInfo.numOutputTensors);
+    graphInfo.m_inputs  = nullptr;
+    graphInfo.m_outputs = nullptr;
+    if (qnn::tools::iotensor::StatusCode::SUCCESS != returnStatus) {
+      QNN_ERROR("Error in tear down Input and output Tensors for graphIdx: %d", graphIdx);
+      break;
+    }
+  }
+
+  return static_cast<sample_app::StatusCode>(returnStatus);
+}
+
 sample_app::StatusCode sample_app::QnnSampleApp::executeGraphsBuffers(std::vector<uint8_t*>& inputBuffers, 
                                                                                std::vector<uint8_t*>& outputBuffers, std::vector<size_t>& outputSize,
                                                                                std::string perfProfile) {
@@ -870,13 +910,10 @@ sample_app::StatusCode sample_app::QnnSampleApp::executeGraphsBuffers(std::vecto
       break;
     }
 
-    Qnn_Tensor_t* inputs  = nullptr;
-    Qnn_Tensor_t* outputs = nullptr;
-    if (iotensor::StatusCode::SUCCESS != m_ioTensor.setupInputAndOutputTensors(&inputs, &outputs, (*m_graphsInfo)[graphIdx])) {
-      QNN_ERROR("Error in setting up Input and output Tensors for graphIdx: %d", graphIdx);
-      returnStatus = StatusCode::FAILURE;
-      break;
-    }
+    // improve performance.
+
+    Qnn_Tensor_t* inputs  = (*m_graphsInfo)[graphIdx].m_inputs ;
+    Qnn_Tensor_t* outputs = (*m_graphsInfo)[graphIdx].m_outputs;
 
     auto graphInfo = (*m_graphsInfo)[graphIdx];
     if (!inputBuffers.empty()) {
@@ -994,12 +1031,6 @@ sample_app::StatusCode sample_app::QnnSampleApp::executeGraphsBuffers(std::vecto
         }
       }
     }
-    m_ioTensor.tearDownInputAndOutputTensors(inputs, outputs, graphInfo.numInputTensors, graphInfo.numOutputTensors);
-    inputs  = nullptr;
-    outputs = nullptr;
-    if (StatusCode::SUCCESS != returnStatus) {
-      break;
-    }
   }
 
   return returnStatus;

diff --git a/src/QnnSampleApp.hpp b/src/QnnSampleApp.hpp
@@ -93,6 +93,9 @@ class QnnSampleApp {
 
   StatusCode verifyFailReturnStatus(Qnn_ErrorHandle_t errCode);
 
+// improve performance.
+  StatusCode setupInputAndOutputTensors();
+  StatusCode tearDownInputAndOutputTensors();
 
 // zw.
   StatusCode executeGraphsBuffers(std::vector<uint8_t*>& inputBuffers,

diff --git a/src/WrapperUtils/QnnWrapperUtils.hpp b/src/WrapperUtils/QnnWrapperUtils.hpp
@@ -122,6 +122,10 @@ typedef struct GraphInfo {
   uint32_t numInputTensors;
   Qnn_Tensor_t *outputTensors;
   uint32_t numOutputTensors;
+
+  // improve performance.
+  Qnn_Tensor_t* m_inputs;
+  Qnn_Tensor_t* m_outputs;
 } GraphInfo_t;
 typedef GraphInfo_t *GraphInfoPtr_t;