Merge pull request #6143 from RasaHQ/fix_empty_strings

Fix empty strings as output of whitespace tokenizer

changelog/6143.bugfix.rst

@@ -0,0 +1 @@
+Prevent ``WhitespaceTokenizer`` from outputting empty list of tokens.

rasa/nlu/tokenizers/whitespace_tokenizer.py

@@ -70,11 +70,11 @@ def tokenize(self, message: Message, attribute: Text) -> List[Token]:
             text,
         ).split()
 
+        words = [self.remove_emoji(w) for w in words]
+        words = [w for w in words if w]
+
         # if we removed everything like smiles `:)`, use the whole text as 1 token
         if not words:
             words = [text]
 
-        words = [self.remove_emoji(w) for w in words]
-        words = [w for w in words if w]
-
         return self._convert_words_to_tokens(words, text)

tests/nlu/tokenizers/test_whitespace_tokenizer.py

@@ -65,6 +65,7 @@
         ),
         (":)", [":)"], [(0, 2)]),
         ("Hi :-)", ["Hi"], [(0, 2)]),
+        ("👍", ["👍"], [(0, 1)]),
     ],
 )
 def test_whitespace(text, expected_tokens, expected_indices):