diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py
index 159ea6fb0be6e..a83e98033bcb9 100755
--- a/python/paddle/fluid/backward.py
+++ b/python/paddle/fluid/backward.py
@@ -2221,6 +2221,10 @@ def _find_op_path_(
                 op.desc.output_arg_names(), output_names
             ):
                 relevant_op_flags[i] = True
+                if core.has_non_empty_grad_op_maker(op.type):
+                    for name in op.desc.input_arg_names():
+                        if name not in no_grad_set:
+                            output_names.add(name)
 
     op_path = [
         block.ops[i] for i in range(len(block.ops)) if relevant_op_flags[i]
diff --git a/python/paddle/fluid/layers/rnn.py b/python/paddle/fluid/layers/rnn.py
index 230f4b5907667..edb526288a96a 100644
--- a/python/paddle/fluid/layers/rnn.py
+++ b/python/paddle/fluid/layers/rnn.py
@@ -694,7 +694,6 @@ def _switch_grad(x, stop=False):
 
     out_array = control_flow.create_array(dtype=flatten(inputs)[0].dtype)
 
-    init_array = control_flow.create_array(dtype=flatten(inputs)[0].dtype)
     init_array = map_structure(
         lambda x: control_flow.create_array(dtype=x.dtype), initial_states
     )