Merge pull request #6119 from RasaHQ/empty_unicode_token

Remove empty tokens from whitespace tokenizer
RasaHQ · Jul 2, 2020 · 3823811 · 3823811
2 parents 70823f9 + 75328fd
commit 3823811
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 0 deletions.
diff --git a/changelog/6119.bugfix.rst b/changelog/6119.bugfix.rst
@@ -0,0 +1 @@
+Explicitly remove all emojis which appear as unicode characters from the output of ``regex.sub`` inside ``WhitespaceTokenizer``.
diff --git a/rasa/nlu/tokenizers/whitespace_tokenizer.py b/rasa/nlu/tokenizers/whitespace_tokenizer.py
@@ -1,6 +1,7 @@
 from typing import Any, Dict, List, Text
 
 import regex
+import re
 
 from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
 from rasa.nlu.training_data import Message
@@ -24,6 +25,27 @@ def __init__(self, component_config: Dict[Text, Any] = None) -> None:
 
         self.case_sensitive = self.component_config["case_sensitive"]
 
+        self.emoji_pattern = self.get_emoji_regex()
+
+    @staticmethod
+    def get_emoji_regex():
+        emoji_pattern = re.compile(
+            "["
+            "\U0001F600-\U0001F64F"  # emoticons
+            "\U0001F300-\U0001F5FF"  # symbols & pictographs
+            "\U0001F680-\U0001F6FF"  # transport & map symbols
+            "\U0001F1E0-\U0001F1FF"  # flags (iOS)
+            "\U00002702-\U000027B0"
+            "\U000024C2-\U0001F251"
+            "]+",
+            flags=re.UNICODE,
+        )
+        return emoji_pattern
+
+    def remove_emoji(self, text: Text) -> Text:
+
+        return self.emoji_pattern.sub(r"", text)
+
     def tokenize(self, message: Message, attribute: Text) -> List[Token]:
         text = message.get(attribute)
 
@@ -47,8 +69,12 @@ def tokenize(self, message: Message, attribute: Text) -> List[Token]:
             " ",
             text,
         ).split()
+
         # if we removed everything like smiles `:)`, use the whole text as 1 token
         if not words:
             words = [text]
 
+        words = [self.remove_emoji(w) for w in words]
+        words = [w for w in words if w]
+
         return self._convert_words_to_tokens(words, text)
diff --git a/tests/nlu/tokenizers/test_whitespace_tokenizer.py b/tests/nlu/tokenizers/test_whitespace_tokenizer.py
@@ -58,6 +58,13 @@
             ],
             [(0, 82), (83, 141)],
         ),
+        (
+            "Joselico gracias Dois 🙏🇺🇸🏦🛠🔥⭐️🦅👑💪",
+            ["Joselico", "gracias", "Dois"],
+            [(0, 8), (9, 16), (17, 21)],
+        ),
+        (":)", [":)"], [(0, 2)]),
+        ("Hi :-)", ["Hi"], [(0, 2)]),
     ],
 )
 def test_whitespace(text, expected_tokens, expected_indices):