Paragraph offsets

tsproisl · Oct 17, 2023 · 3c665c5 · 3c665c5
1 parent 6a6d51c
commit 3c665c5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/src/somajo/tokenizer.py b/src/somajo/tokenizer.py
@@ -846,6 +846,7 @@ def tokenize_file(self, filename, parsep_empty_lines=True):
             if parsep_empty_lines:
                 parsep = "empty_lines"
             paragraphs = utils.get_paragraphs_str(f, paragraph_separator=parsep)
+            paragraphs = (paragraph for paragraph, position in paragraphs)
             tokenized_paragraphs = map(self.tokenize_paragraph, paragraphs)
             for tp in tokenized_paragraphs:
                 if tp:

diff --git a/src/somajo/utils.py b/src/somajo/utils.py
@@ -11,31 +11,35 @@
 
 def get_paragraphs_str(fh, paragraph_separator="empty_lines"):
     """Generator for the paragraphs in the file."""
+    position = 0
     if paragraph_separator == "single_newlines":
         for line in fh:
             if line.strip() != "":
-                yield line
+                yield line, position
+            position += len(line)
     elif paragraph_separator == "empty_lines":
         paragraph = []
         for line in fh:
             if line.strip() == "":
                 if len(paragraph) > 0:
-                    yield "".join(paragraph)
+                    paragraph_text = "".join(paragraph)
+                    yield paragraph_text, position
                     paragraph = []
+                    position += len(paragraph_text)
             else:
                 paragraph.append(line)
         if len(paragraph) > 0:
-            yield "".join(paragraph)
+            yield "".join(paragraph), position
 
 
 def get_paragraphs_list(text_file, paragraph_separator="empty_lines"):
     """Generator for the paragraphs in the file."""
     if isinstance(text_file, str):
         with open(text_file, encoding="utf-8") as fh:
-            for paragraph in get_paragraphs_str(fh, paragraph_separator):
+            for paragraph, position in get_paragraphs_str(fh, paragraph_separator):
                 yield [Token(paragraph, first_in_sentence=True, last_in_sentence=True)]
     else:
-        for paragraph in get_paragraphs_str(text_file, paragraph_separator):
+        for paragraph, position in get_paragraphs_str(text_file, paragraph_separator):
             yield [Token(paragraph, first_in_sentence=True, last_in_sentence=True)]