【Qwen2-VL Inference】add qwen2-vl high performance inference

paddlenlp/experimental/transformers/fused_transformer_layers.py

@@ -1067,6 +1067,14 @@ def forward(
         residual_input = src
         for i in range(self.num_layers):
             qkv_out, residual_input = self.compute_qkv(src, residual_input, i)
+
+            import paddle
+
+            to_file = "/root/tmp/rotary.pdparams"
+            state_dict = paddle.load(to_file, return_numpy=False)
+            rotary_embs = state_dict["inputs_embeds"].cast("float32")
+            # print(rotary_embs.shape)
+
             out_linear_out = self.compute_attn(
                 time_step,
                 qkv_out,

paddlenlp/experimental/transformers/generation_utils.py

@@ -743,12 +743,13 @@ def _post_process_(
                     model_kwargs["next_tokens"],
                 )
 
-                save_output(
-                    next_tokens,
-                    model_kwargs["not_need_stop"],
-                    model_kwargs.get("accept_num", None),  # only initialized in speculative decoding
-                    self.config.tensor_parallel_rank,
-                )
+                # save_output(
+                #     next_tokens,
+                #     model_kwargs["not_need_stop"],
+                #     model_kwargs.get("accept_num", None),  # only initialized in speculative decoding
+                #     self.config.tensor_parallel_rank,
+                # )
+
                 return next_tokens
             else:
                 from paddlenlp_ops import (