fix: add keep_empty to st clean_text

engisalor · Jul 16, 2024 · 398a371 · 398a371
1 parent 81bf898
commit 398a371
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/pipeline/stanza/base_pipeline.py b/pipeline/stanza/base_pipeline.py
@@ -74,10 +74,10 @@ def wrap_lines(match, hunspell: bool = False, dictionaries="en_US,es_ES,fr_FR"):
         return match.group()
 
 
-def clean_text(text: str) -> str:
+def clean_text(text: str, keep_empty=False) -> str:
     """Cleans texts to prepare for passing to an NLP pipeline."""
     lines = text.split("\n")
-    lines = [uninorm.normalize_line(x) for x in lines]
+    lines = [uninorm.normalize_line(x, keep_empty=keep_empty) for x in lines]
     return "".join(lines)