Find a robust method to get articles paragraphs #1

cdqa-suite · Feb 19, 2019 · ec0f199 · ec0f199
1 parent 4b5ba92
commit ec0f199
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/examples/run_converter.py b/examples/run_converter.py
@@ -1,8 +1,10 @@
 import pandas as pd
-from cdqa.utils.converter import df2squad
+from cdqa.utils.converter import df2squad, filter_paragraphs
 from ast import literal_eval
 
 # https://stackoverflow.com/questions/32742976/how-to-read-a-column-of-csv-as-dtype-list-using-pandas
 df = pd.read_csv('bnpp_newsroom_v1.0.csv', converters={'paragraphs': literal_eval})
 
+df['paragraphs'] = df['paragraphs'].apply(filter_paragraphs)
+
 json_data = df2squad(df=df, version='v2.0', output_dir='./')