diff --git a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
index 75e82d7f195a6..e8fe4ce62e7f1 100644
--- a/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
+++ b/paddle/fluid/framework/ir/mkldnn/cpu_quantize_squash_pass.cc
@@ -113,15 +113,19 @@ bool CPUQuantizeSquashPass::IsDequantizeInputUint8(
   if (dequant_in->inputs[0]->IsOp()) {
     auto prev_op = dequant_in->inputs[0]->Op();
     std::string act_name;
-    if (prev_op->Type() == "conv2d") {
-      act_name = "fuse_activation";
-    } else if (prev_op->Type() == "fc") {
-      act_name = "activation_type";
-    }
-    if (!act_name.empty()) {
-      auto act = prev_op->GetAttrIfExists<std::string>(act_name);
-      if (act == "relu" || act == "relu6") {
-        return true;
+    if (prev_op->Type() == "relu") {
+      return true;
+    } else {
+      if (prev_op->Type() == "conv2d") {
+        act_name = "fuse_activation";
+      } else if (prev_op->Type() == "fc") {
+        act_name = "activation_type";
+      }
+      if (!act_name.empty()) {
+        auto act = prev_op->GetAttrIfExists<std::string>(act_name);
+        if (act == "relu" || act == "relu6") {
+          return true;
+        }
       }
     }
   }