From 9a8509268bf761b7d5a53625806b23d77be5cdbe Mon Sep 17 00:00:00 2001 From: Habeeb Shopeju Date: Sun, 6 Feb 2022 13:50:37 +0000 Subject: [PATCH 1/2] Cleaned up path appends to current directory --- scripts/ltr_msmarco/convert_passage.py | 2 -- scripts/ltr_msmarco/convert_passage_doc.py | 2 -- scripts/ltr_msmarco/convert_queries.py | 1 - 3 files changed, 5 deletions(-) diff --git a/scripts/ltr_msmarco/convert_passage.py b/scripts/ltr_msmarco/convert_passage.py index f1f51ad6a..74026280b 100644 --- a/scripts/ltr_msmarco/convert_passage.py +++ b/scripts/ltr_msmarco/convert_passage.py @@ -16,7 +16,6 @@ import multiprocessing from joblib import Parallel, delayed -import sys import json import argparse from transformers import AutoTokenizer, AutoModel @@ -29,7 +28,6 @@ """ add fields to jsonl with text(lemmatized), text_unlemm, contents(analyzer), raw, text_bert_tok(BERT token) """ -sys.path.append('.') parser = argparse.ArgumentParser(description='Convert MSMARCO-adhoc documents.') parser.add_argument('--input', metavar='input file', help='input file', diff --git a/scripts/ltr_msmarco/convert_passage_doc.py b/scripts/ltr_msmarco/convert_passage_doc.py index ae4a585b4..675e7ec74 100644 --- a/scripts/ltr_msmarco/convert_passage_doc.py +++ b/scripts/ltr_msmarco/convert_passage_doc.py @@ -16,7 +16,6 @@ import multiprocessing from joblib import Parallel, delayed -import sys import json import argparse from transformers import AutoTokenizer, AutoModel @@ -29,7 +28,6 @@ """ add fields to jsonl with text(lemmatized), text_unlemm, contents(analyzer), raw, text_bert_tok(BERT token) """ -sys.path.append('.') parser = argparse.ArgumentParser(description='Convert MSMARCO-adhoc documents.') parser.add_argument('--input', metavar='input file', help='input file', diff --git a/scripts/ltr_msmarco/convert_queries.py b/scripts/ltr_msmarco/convert_queries.py index 9c8be5ea0..4496b55e0 100644 --- a/scripts/ltr_msmarco/convert_queries.py +++ b/scripts/ltr_msmarco/convert_queries.py @@ -33,7 +33,6 @@ """ add fields to query json with text(lemmatized), text_unlemm, contents(analyzer), raw, entity(NER), text_bert_tok(BERT token) """ -sys.path.append('.') parser = argparse.ArgumentParser(description='Convert MSMARCO-adhoc queries.') parser.add_argument('--input', metavar='input file', help='input file', From 437f1a96a3553f86d12c6274a6166cec22cbcebe Mon Sep 17 00:00:00 2001 From: Habeeb Shopeju Date: Sun, 6 Feb 2022 17:19:20 +0000 Subject: [PATCH 2/2] Cleaned up path insert --- scripts/ltr_msmarco/convert_queries.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/ltr_msmarco/convert_queries.py b/scripts/ltr_msmarco/convert_queries.py index 4496b55e0..f96fd1c96 100644 --- a/scripts/ltr_msmarco/convert_queries.py +++ b/scripts/ltr_msmarco/convert_queries.py @@ -16,12 +16,6 @@ """Convert MSMARCO queries""" -import sys - -# We're going to explicitly use a local installation of Pyserini (as opposed to a pip-installed one). -# Comment these lines out to use a pip-installed one instead. -sys.path.insert(0, './') - import json import argparse from transformers import AutoTokenizer, AutoModel