From c2084b34e852ac1de79ce08b2cc8cb457d7e4355 Mon Sep 17 00:00:00 2001 From: smallv0221 <397551318@qq.com> Date: Thu, 20 Jan 2022 11:45:43 +0000 Subject: [PATCH] fix truncation_strategy for tokenizer --- paddlenlp/transformers/tokenizer_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddlenlp/transformers/tokenizer_utils.py b/paddlenlp/transformers/tokenizer_utils.py index d820a618a94a..e915fb09875d 100644 --- a/paddlenlp/transformers/tokenizer_utils.py +++ b/paddlenlp/transformers/tokenizer_utils.py @@ -616,7 +616,7 @@ def __call__(self, stride=stride, is_split_into_words=is_split_into_words, pad_to_max_seq_len=pad_to_max_seq_len, - truncation_strategy="longest_first", + truncation_strategy=truncation_strategy, return_position_ids=return_position_ids, return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask, @@ -629,7 +629,7 @@ def __call__(self, text_pair=text_pair, max_seq_len=max_seq_len, pad_to_max_seq_len=pad_to_max_seq_len, - truncation_strategy="longest_first", + truncation_strategy=truncation_strategy, return_position_ids=return_position_ids, return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask,