Find a robust method to get articles paragraphs #1

cdqa-suite · Feb 19, 2019 · 4b5ba92 · 4b5ba92
1 parent ab2e007
commit 4b5ba92
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/cdqa/utils/converter.py b/cdqa/utils/converter.py
@@ -34,3 +34,8 @@ def df2squad(df, version='v2.0', output_dir=None):
            json.dump(json_data, outfile)
 
    return json_data
+
+def filter_paragraphs(paragraphs):
+    # filter out paragraphs shorter than 10 words and longer than 250 words
+    paragraphs_filtered = [paragraph for paragraph in paragraphs if len(paragraph.split()) >= 10 and len(paragraph.split()) <= 250]
+    return paragraphs_filtered