[duuie] fix max_seq_len bug in duuie code

examples/information_extraction/DuUIE/uie/evaluation/sel2record.py

@@ -764,9 +764,9 @@ def clean_wildcard(x):
 def fix_unk_from_text_with_tokenizer(span, text, tokenizer, unk='<unk>'):
     unk_id = tokenizer.vocab.to_indices(unk)
     tokenized_span = tokenizer.encode(
-        span, add_special_tokens=False)['input_ids']
+        span, add_special_tokens=False, return_token_type_ids=None)['input_ids']
     tokenized_text = tokenizer.encode(
-        text, add_special_tokens=False)['input_ids']
+        text, add_special_tokens=False, return_token_type_ids=None)['input_ids']
 
     matched = match_sublist(tokenized_text, tokenized_span)
     if len(matched) == 0:

examples/information_extraction/DuUIE/uie/seq2struct/data_collator.py

@@ -114,11 +114,10 @@ def get_ordered_dict(schema_name_list, tokenizer):
         """
         schema_ordered_dict = OrderedDict()
         for name in schema_name_list:
-            # tokenizer.encode("人物")
-            encoded_name = tokenizer.encode(name)
-            # {'input_ids': [8, 122, 1]}
-            #   -> [8, 122]
-            schema_ordered_dict[name] = encoded_name["input_ids"][:-1:]
+            # tokenizer.encode("人物") -> [8, 122]
+            encoded_name = tokenizer.encode(
+                name, add_special_tokens=False, return_token_type_ids=None)
+            schema_ordered_dict[name] = encoded_name["input_ids"]
         return schema_ordered_dict
 
     @staticmethod

examples/information_extraction/DuUIE/uie/seq2struct/t5_bert_tokenizer.py

@@ -40,6 +40,8 @@ def __init__(self,
             tokenize_chinese_chars=tokenize_chinese_chars,
             strip_accents=strip_accents,
             **kwargs, )
+        if space_token not in self._additional_special_tokens:
+            self._additional_special_tokens += [space_token]
 
         self._space_token = space_token
 
@@ -51,7 +53,7 @@ def get_vocab(self):
         vocab.update(self.added_tokens_encoder)
         return vocab
 
-    def tokenize(self, text):
+    def tokenize(self, text, **kwargs):
         import re
         # Remove space between <extra_id_*> <spot> <asoc>
         split_bracket = re.compile(
@@ -64,7 +66,7 @@ def tokenize(self, text):
                 new_text_list += [item[0].strip(), item[1]]
             text = "".join(new_text_list)
         text = text.replace(' ', self._space_token)
-        return super().tokenize(text)
+        return super().tokenize(text, **kwargs)
 
     def _add_eos_if_not_present(self, token_ids: List[int]) -> List[int]:
         """Do not add eos again if user already added it."""