From 7f6efa31a5c14290aa724ae4522b29095b53ebf5 Mon Sep 17 00:00:00 2001 From: Ryan Clancy Date: Fri, 28 Jun 2019 13:57:54 -0400 Subject: [PATCH] Merge azure-script into master (#113) * Update Azure script * and make sure it's uncommented... * Enable timings for terrier and pisa * Capture logs and scp back to local. * Update robust04 runbook * Remove P@5 from OldDog config * robust04, core17, and core18 runbooks. * Update terrier version in runbook * Update core17 runbook * Update terrier runbook * Whoops... * Update olddog version * Add anserini-bm25prf to runbook * Update anserini-bm25prf * Update anserini-bm25prf * Update robust05 path for irc-centre2019 * Add ielab and galago to runbook * Write an additional file with the trec_eval results * Update runbook for official releases * Add NDCG@20 by default and fix anserini-bm25prf output directory * Don't create .trec_eval files for non-trec files * Merge master into azure-script (#111) * Add files via upload (#104) * Evaluate on all topics instead of those with documents retrieved (#108) * Added TREC7 and TREC8 qrels (primarily to avoid confusion with the WT qrels in `qrels/qrels.401-450.txt` * Missing TREC7 topics files * Change default to setting also used in the "official" evaluation * Add -c flag for trec_eval * Revert "Change default to setting also used in the "official" evaluation" This reverts commit 85b13dfc48d98593aa1530b97b6b5493ceecfd07. * TREC Terabyte 04-06 topic/qrel files merged (#109) --- azure/azure.sh | 32 ++++--- azure/osirrc2019.json | 79 ---------------- azure/osirrc2019/core17.json | 93 +++++++++++++++++++ azure/osirrc2019/core18.json | 101 +++++++++++++++++++++ azure/osirrc2019/robust04.json | 159 +++++++++++++++++++++++++++++++++ run.py | 45 ++++------ searcher.py | 20 +++-- 7 files changed, 405 insertions(+), 124 deletions(-) delete mode 100644 azure/osirrc2019.json create mode 100644 azure/osirrc2019/core17.json create mode 100644 azure/osirrc2019/core18.json create mode 100644 azure/osirrc2019/robust04.json diff --git a/azure/azure.sh b/azure/azure.sh index 0bd500e..101f64f 100755 --- a/azure/azure.sh +++ b/azure/azure.sh @@ -98,6 +98,7 @@ ssh jig@${IP_ADDRESS} << EOF git clone https://github.com/osirrc2019/jig.git; cd jig + mkdir logs virtualenv -p /usr/bin/python3 venv source venv/bin/activate @@ -120,30 +121,39 @@ TOPIC_PATH=$(cat ${RUN_FILE} | jq -r ".topic.path") QRELS_PATH=$(cat ${RUN_FILE} | jq -r ".qrels.path") # The number of images -NUM_IMAGES=$(cat ${RUN_FILE} | jq -r '.images | length') +NUM_IMAGES=$(cat ${RUN_FILE} | jq -r ".images | length") for i in $(seq 0 $((${NUM_IMAGES} - 1))); do + NAME=$(cat ${RUN_FILE} | jq -r ".images[$i].name") + # Get the search command and substitute in variables PREPARE=$(cat ${RUN_FILE} | jq -r ".images[$i].command.prepare") PREPARE=${PREPARE/"[COLLECTION_NAME]"/${COLLECTION_NAME}} PREPARE=${PREPARE/"[COLLECTION_PATH]"/${COLLECTION_PATH}} PREPARE=${PREPARE/"[COLLECTION_FORMAT]"/${COLLECTION_FORMAT}} - # Get the search command and substitute in variables - SEARCH=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search") - SEARCH=${SEARCH/"[COLLECTION_NAME]"/${COLLECTION_NAME}} - SEARCH=${SEARCH/"[TOPIC_PATH]"/${TOPIC_PATH}} - SEARCH=${SEARCH/"[QRELS_PATH]"/${QRELS_PATH}} - SEARCH=${SEARCH/"[OUTPUT]"/${OUTPUT}} - echo ${PREPARE} - ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && eval ${PREPARE}" + ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && ${PREPARE} > logs/${NAME}-${COLLECTION_NAME}-prepare.log" + + NUM_SEARCHES=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search | length") + + for j in $(seq 0 $((${NUM_SEARCHES} - 1))); do + + # Get the search command and substitute in variables + SEARCH=$(cat ${RUN_FILE} | jq -r ".images[$i].command.search[$j]") + SEARCH=${SEARCH/"[COLLECTION_NAME]"/${COLLECTION_NAME}} + SEARCH=${SEARCH/"[TOPIC_PATH]"/${TOPIC_PATH}} + SEARCH=${SEARCH/"[QRELS_PATH]"/${QRELS_PATH}} + SEARCH=${SEARCH/"[OUTPUT]"/${OUTPUT}} + + echo ${SEARCH} + ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && ${SEARCH} > logs/${NAME}-${COLLECTION_NAME}-search-${j}.log" - echo ${SEARCH} - ssh jig@${IP_ADDRESS} "cd jig && source venv/bin/activate && eval ${SEARCH}" + done scp -r jig@${IP_ADDRESS}:${OUTPUT} . + scp -r jig@${IP_ADDRESS}:/home/jig/jig/logs . done diff --git a/azure/osirrc2019.json b/azure/osirrc2019.json deleted file mode 100644 index 7dcbe88..0000000 --- a/azure/osirrc2019.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "collection": { - "name": "robust04", - "path": "/home/jig/collections/disk45", - "format": "trectext" - }, - "topic": { - "path": "topics/topics.robust04.txt" - }, - "qrels": { - "path": "qrels/qrels.robust2004.txt" - }, - "output": "/tmp/output", - "images": [ - { - "name": "anserini", - "command": { - "prepare": "python run.py prepare --repo osirrc2019/anserini --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python run.py search --repo osirrc2019/anserini --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts search_args=\"-bm25\" out_file_name=\"run.bm25.robust04\"" - } - }, - { - "name": "atire", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/atire --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/atire --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --top_k 100 --output [OUTPUT]/atire --qrels [QRELS_PATH]" - } - }, - { - "name": "ielab", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/ielab --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/ielab --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --output [OUTPUT]/ielab --qrels [QRELS_PATH]" - } - }, - { - "name": "irc-centre2019", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/irc-centre2019 --collections core17=/home/jig/collections/disk45=trectext robust04=/home/jig/collections/disk45=trectext robust05=/home/jig/collections/disk45=trectext --opts run=\"wcrobust04\"", - "search": "python3 run.py search --repo osirrc2019/irc-centre2019 --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --output [OUTPUT]/ielab --qrels [QRELS_PATH]" - } - }, - { - "name": "jass", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/jass --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/jass --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --top_k 100 --output [OUTPUT]/jass --qrels [QRELS_PATH]" - } - }, - { - "name": "jassv2", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/jassv2 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/jassv2 --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --top_k 100 --output [OUTPUT]/jassv2 --qrels [QRELS_PATH]" - } - }, - { - "name": "olddog", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/olddog --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/olddog --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --output [OUTPUT]/olddog --qrels [QRELS_PATH] --opts out_file_name=\"run.bm25.robust04\"" - } - }, - { - "name": "pisa", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/pisa --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/pisa --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --output [OUTPUT]/pisa --qrels [QRELS_PATH]" - } - }, - { - "name": "terrier", - "command": { - "prepare": "python3 run.py prepare --repo osirrc2019/terrier --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", - "search": "python3 run.py search --repo osirrc2019/terrier --collection [COLLECTION_NAME] --topic [TOPIC_PATH] --output [OUTPUT]/terrier --qrels [QRELS_PATH]" - } - } - ] -} \ No newline at end of file diff --git a/azure/osirrc2019/core17.json b/azure/osirrc2019/core17.json new file mode 100644 index 0000000..5563594 --- /dev/null +++ b/azure/osirrc2019/core17.json @@ -0,0 +1,93 @@ +{ + "collection": { + "name": "core17", + "path": "/home/jig/collections/NYTcorpus", + "format": "trectext" + }, + "topic": { + "path": "topics/topics.core17.txt" + }, + "qrels": { + "path": "qrels/qrels.core17.txt" + }, + "output": "/tmp/output", + "images": [ + { + "name": "anserini", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/anserini --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.run\" search_args=\"-bm25\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.rm3.run\" search_args=\"-bm25 -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.ax.run\" search_args=\"-bm25 -axiom -rerankCutoff 20 -axiom.deterministic\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.run\" search_args=\"-ql\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.rm3.run\" search_args=\"-ql -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.ax.run\" search_args=\"-ql -axiom -rerankCutoff 20 -axiom.deterministic\"" + ] + } + }, + { + "name": "atire", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/atire --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/atire --tag v0.1.1 --output [OUTPUT]/atire --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "ielab", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/ielab --tag v0.0.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/ielab --tag v0.0.1 --output [OUTPUT]/ielab --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "irc-centre2019-wcrobust04", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/irc-centre2019 --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] robust04=/home/jig/collections/disk45=trectext --opts run=\"wcrobust04\"", + "search": [ + "python run.py search --repo osirrc2019/irc-centre2019 --tag v0.1.3 --output [OUTPUT]/irc-centre2019 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "irc-centre2019-wcrobust0405", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/irc-centre2019 --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] robust04=/home/jig/collections/disk45=trectext robust05=/home/jig/collections/AQUAINT=trectext --opts run=\"wcrobust0405\"", + "search": [ + "python run.py search --repo osirrc2019/irc-centre2019 --tag v0.1.3 --output [OUTPUT]/irc-centre2019 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "jass", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/jass --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/jass --tag v0.1.1 --output [OUTPUT]/jass --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "jassv2", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/jassv2 --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/jassv2 --tag v0.1.1 --output [OUTPUT]/jassv2 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "pisa", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/pisa --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/pisa --tag v0.1.3 --output [OUTPUT]/pisa --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + } + ] +} \ No newline at end of file diff --git a/azure/osirrc2019/core18.json b/azure/osirrc2019/core18.json new file mode 100644 index 0000000..c6acc7d --- /dev/null +++ b/azure/osirrc2019/core18.json @@ -0,0 +1,101 @@ +{ + "collection": { + "name": "core18", + "path": "/home/jig/collections/WashingtonPost.v2", + "format": "json" + }, + "topic": { + "path": "topics/topics.core18.txt" + }, + "qrels": { + "path": "qrels/qrels.core18.txt" + }, + "output": "/tmp/output", + "images": [ + { + "name": "anserini", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/anserini --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.run\" search_args=\"-bm25\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.rm3.run\" search_args=\"-bm25 -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.ax.run\" search_args=\"-bm25 -axiom -rerankCutoff 20 -axiom.deterministic\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.run\" search_args=\"-ql\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.rm3.run\" search_args=\"-ql -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.ax.run\" search_args=\"-ql -axiom -rerankCutoff 20 -axiom.deterministic\"" + ] + } + }, + { + "name": "ielab", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/ielab --tag v0.0.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/ielab --tag v0.0.1 --output [OUTPUT]/ielab --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "indri", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/indri --tag v0.2.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.dir1000.title\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.dir1000.title.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.dir1000.title.prf\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" use_prf=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.dir1000.title.prf.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" use_prf=\"1\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.jm0.5.title\" rule=\"method:linear,collectionLambda:0.5\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.bm25.title\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.bm25.title.prf\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title\" use_prf=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.bm25.title+desc\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title+desc\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.bm25.title+desc.prf.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title+desc\" use_prf=\"1\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"core18.dir1000.desc\" rule=\"method:dirichlet,mu:1000\" topic_type=\"desc\"" + ] + } + }, + { + "name": "olddog", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/olddog --tag v1.0.0 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/pisa --olddog v1.0.0 --output [OUTPUT]/pisa --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings" + ] + } + }, + { + "name": "pisa", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/pisa --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/pisa --tag v0.1.3 --output [OUTPUT]/pisa --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings" + ] + } + }, + { + "name": "terrier", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/terrier --tag v0.1.7 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=pl2", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=pl2_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_qe" + ] + } + }, + { + "name": "terrier-block", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/terrier --tag v0.1.7 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] --opts block.indexing=true", + "search": [ + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_prox", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_prox_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_prox", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_prox_qe" + ] + } + } + ] +} \ No newline at end of file diff --git a/azure/osirrc2019/robust04.json b/azure/osirrc2019/robust04.json new file mode 100644 index 0000000..6a82507 --- /dev/null +++ b/azure/osirrc2019/robust04.json @@ -0,0 +1,159 @@ +{ + "collection": { + "name": "robust04", + "path": "/home/jig/collections/disk45", + "format": "trectext" + }, + "topic": { + "path": "topics/topics.robust04.txt" + }, + "qrels": { + "path": "qrels/qrels.robust04.txt" + }, + "output": "/tmp/output", + "images": [ + { + "name": "anserini", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/anserini --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.run\" search_args=\"-bm25\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.rm3.run\" search_args=\"-bm25 -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"bm25.ax.run\" search_args=\"-bm25 -axiom -rerankCutoff 20 -axiom.deterministic\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.run\" search_args=\"-ql\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.rm3.run\" search_args=\"-ql -rm3\"", + "python run.py search --repo osirrc2019/anserini --tag v0.1.1 --output [OUTPUT]/anserini --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts out_file_name=\"ql.ax.run\" search_args=\"-ql -axiom -rerankCutoff 20 -axiom.deterministic\"" + ] + } + }, + { + "name": "anserini-bm25prf", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/anserini-bm25prf --tag v0.2.2 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/anserini-bm25prf --tag v0.2.2 --output [OUTPUT]/anserini-bm25prf --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings", + "python run.py search --repo osirrc2019/anserini-bm25prf --tag v0.2.2 --output [OUTPUT]/anserini-bm25prf --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts k1=0.9 b=0.2 k1_prf=0.9 b_prf=0.6 num_new_terms=40 num_docs=10 new_term_weight=0.1 " + ] + } + }, + { + "name": "atire", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/atire --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/atire --tag v0.1.1 --output [OUTPUT]/atire --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "birch", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/birch --tag v0.1.0 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/birch --tag v0.1.0 --output [OUTPUT]/birch --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --measures map P.20 --opts experiment=qa_2cv num_folds=2 anserini_path=anserini tune_params=False", + "python run.py search --repo osirrc2019/birch --tag v0.1.0 --output [OUTPUT]/birch --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --measures map P.20 --opts experiment=mb_2cv num_folds=2 anserini_path=anserini tune_params=False", + "python run.py search --repo osirrc2019/birch --tag v0.1.0 --output [OUTPUT]/birch --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --measures map P.20 --opts experiment=qa_5cv num_folds=5 anserini_path=anserini tune_params=False", + "python run.py search --repo osirrc2019/birch --tag v0.1.0 --output [OUTPUT]/birch --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --measures map P.20 --opts experiment=mb_5cv num_folds=5 anserini_path=anserini tune_params=False" + ] + } + }, + { + "name": "galago", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/galago --tag v0.0.2 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/galago --tag v0.0.2 --output [OUTPUT]/galago --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "ielab", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/ielab --tag v0.0.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/ielab --tag v0.0.1 --output [OUTPUT]/ielab --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME]" + ] + } + }, + { + "name": "indri", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/indri --tag v0.2.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.dir1000.title\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.dir1000.title.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.dir1000.title.prf\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" use_prf=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.dir1000.title.prf.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title\" use_prf=\"1\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.jm0.5.title\" rule=\"method:linear,collectionLambda:0.5\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.bm25.title\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.bm25.title.prf\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title\" use_prf=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.bm25.title+desc\" rule=\"okapi,k1:1.2,b:0.75\" topic_type=\"title+desc\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.bm25.title+desc.prf.sd\" rule=\"method:dirichlet,mu:1000\" topic_type=\"title+desc\" use_prf=\"1\" sd=\"1\"", + "python run.py search --repo osirrc2019/indri --tag v0.2.1 --output [OUTPUT]/indri --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"robust.dir1000.desc\" rule=\"method:dirichlet,mu:1000\" topic_type=\"desc\"" + ] + } + }, + { + "name": "jass", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/jass --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/jass --tag v0.1.1 --output [OUTPUT]/jass --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "jassv2", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/jassv2 --tag v0.1.1 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/jassv2 --tag v0.1.1 --output [OUTPUT]/jassv2 --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --top_k 100 --timings" + ] + } + }, + { + "name": "olddog", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/olddog --tag v1.0.0 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/olddog --tag v1.0.0 --output [OUTPUT]/olddog --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --opts out_file_name=\"run.bm25.robust04\"" + ] + } + }, + { + "name": "pisa", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/pisa --tag v0.1.3 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/pisa --tag v0.1.3 --output [OUTPUT]/pisa --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings" + ] + } + }, + { + "name": "terrier", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/terrier --tag v0.1.7 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT]", + "search": [ + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=pl2", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=pl2_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_qe" + ] + } + }, + { + "name": "terrier-block", + "command": { + "prepare": "python run.py prepare --repo osirrc2019/terrier --tag v0.1.7 --collections [COLLECTION_NAME]=[COLLECTION_PATH]=[COLLECTION_FORMAT] --opts block.indexing=true", + "search": [ + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_prox", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=bm25_prox_qe", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_prox", + "python run.py search --repo osirrc2019/terrier --tag v0.1.7 --output [OUTPUT]/terrier --qrels [QRELS_PATH] --topic [TOPIC_PATH] --collection [COLLECTION_NAME] --timings --opts config=dph_prox_qe" + ] + } + } + ] +} diff --git a/run.py b/run.py index 24c191b..6dfaef3 100644 --- a/run.py +++ b/run.py @@ -24,9 +24,8 @@ def str_to_bool(s): parser_prepare.set_defaults(run=manager.prepare) parser_prepare.add_argument("--repo", required=True, type=str, help="the image repo (i.e., osirrc2019/anserini)") parser_prepare.add_argument("--tag", default="latest", type=str, help="the image tag (i.e., latest)") - parser_prepare.add_argument("--save_to_snapshot", default="save", type=str, - help="used to determine the tag of the snapshotted image after indexing") - parser_prepare.add_argument("--collections", required=True, nargs="+", help="the name of the collection") + parser_prepare.add_argument("--save_to_snapshot", default="save", type=str, help="used to determine the tag of the snapshotted image after indexing") + parser_prepare.add_argument("--collections", nargs="+", help="the name of the collection") parser_prepare.add_argument("--opts", nargs="+", default="", type=str, help="the args passed to the index script") # Specific to train @@ -34,19 +33,14 @@ def str_to_bool(s): trainer_prepare.set_defaults(run=manager.train) trainer_prepare.add_argument("--repo", required=True, type=str, help="the image repo (i.e., albep/nvsm)") trainer_prepare.add_argument("--tag", default="latest", type=str, help="the image tag (i.e., latest)") - trainer_prepare.add_argument("--load_from_snapshot", default="save", type=str, - help="used to determine the tag of the snapshotted image to train from") + trainer_prepare.add_argument("--load_from_snapshot", default="save", type=str, help="used to determine the tag of the snapshotted image to train from") trainer_prepare.add_argument("--topic", required=True, type=str, help="the topic file for search") - trainer_prepare.add_argument("--collection", required=True, help="the name of the collection for train") + trainer_prepare.add_argument("--collection", help="the name of the collection for train") trainer_prepare.add_argument("--topic_format", default="trec", type=str, help="the topic file format for training") - trainer_prepare.add_argument("--save_to_snapshot", default="save", type=str, - help="used to determine the tag of the snapshotted image after indexing") - trainer_prepare.add_argument("--test_split", required=True, type=str, - help="the subset of topic ids to use for testing") - trainer_prepare.add_argument("--validation_split", required=True, type=str, - help="the subset of topic ids to use for validation") - trainer_prepare.add_argument("--model_folder", required=True, type=str, - help="folder where to store the trained model") + trainer_prepare.add_argument("--save_to_snapshot", default="save", type=str, help="used to determine the tag of the snapshotted image after indexing") + trainer_prepare.add_argument("--test_split", required=True, type=str, help="the subset of topic ids to use for testing") + trainer_prepare.add_argument("--validation_split", required=True, type=str, help="the subset of topic ids to use for validation") + trainer_prepare.add_argument("--model_folder", required=True, type=str, help="folder where to store the trained model") trainer_prepare.add_argument("--qrels", required=True, type=str, help="the qrels file for model selection") trainer_prepare.add_argument("--opts", nargs="+", default="", type=str, help="the args passed to the index script") trainer_prepare.add_argument("--version", default="", type=str, help="the version string passed to the init script") @@ -56,34 +50,27 @@ def str_to_bool(s): parser_search.set_defaults(run=manager.search) parser_search.add_argument("--repo", required=True, type=str, help="the image repo (i.e., osirrc2019/anserini)") parser_search.add_argument("--tag", default="latest", type=str, help="the image tag (i.e., latest)") - parser_search.add_argument("--load_from_snapshot", default="save", type=str, - help="used to determine the tag of the snapshotted image to search from") + parser_search.add_argument("--load_from_snapshot", default="save", type=str, help="used to determine the tag of the snapshotted image to search from") parser_search.add_argument("--collection", required=True, help="the name of the collection") parser_search.add_argument("--topic", required=True, type=str, help="the topic file for search") parser_search.add_argument("--topic_format", default="trec", type=str, help="the topic file format for search") parser_search.add_argument("--top_k", default=1000, type=int, help="the number of results for top-k retrieval") - parser_search.add_argument("--output", required=True, type=str, - help="the output directory for run files on the host") + parser_search.add_argument("--output", required=True, type=str, help="the output directory for run files on the host") parser_search.add_argument("--qrels", required=True, type=str, help="the qrels file for evaluation") - parser_search.add_argument("--test_split", required=False, default="", type=str, - help="the subset of topic ids to use for testing") + parser_search.add_argument("--test_split", required=False, default="", type=str, help="the subset of topic ids to use for testing") parser_search.add_argument("--opts", nargs="+", default="", type=str, help="the args passed to the search script") parser_search.add_argument("--timings", action="store_true", help="enable timing information to be printed") - parser_search.add_argument("--measures", nargs="+", default=["num_q", "map", "P.30"], type=str, help="the measures for trec_eval") + parser_search.add_argument("--measures", nargs="+", default=["num_q", "map", "P.30", "ndcg_cut.20"], type=str, help="the measures for trec_eval") # Specific to interact parser_interact = parser_sub.add_parser("interact") parser_interact.set_defaults(run=manager.interact) parser_interact.add_argument("--repo", required=True, type=str, help="the image repo (i.e., osirrc2019/anserini)") parser_interact.add_argument("--tag", default="latest", type=str, help="the image tag (i.e., latest)") - parser_interact.add_argument("--load_from_snapshot", default="save", type=str, - help="used to determine the tag of the snapshotted image to interact with") - parser_interact.add_argument("--exit_jig", default="false", type=str_to_bool, - help="whether to exit jig after running container") - parser_interact.add_argument("--opts", nargs="+", default="", type=str, - help="the args passed to the interact script") + parser_interact.add_argument("--load_from_snapshot", default="save", type=str, help="used to determine the tag of the snapshotted image to interact with") + parser_interact.add_argument("--exit_jig", default="false", type=str_to_bool, help="whether to exit jig after running container") + parser_interact.add_argument("--opts", nargs="+", default="", type=str, help="the args passed to the interact script") # Parse the args args = parser.parse_args() - - args.run(args) + args.run(args) \ No newline at end of file diff --git a/searcher.py b/searcher.py index 9ef665f..3101f64 100644 --- a/searcher.py +++ b/searcher.py @@ -25,8 +25,12 @@ def search(self, client, output_path_guest, topic_path_guest, test_split_path_gu topic_path_host = os.path.dirname(os.path.abspath(self.config.topic)) + output_path = os.path.abspath(self.config.output) + if not os.path.exists(output_path): + os.makedirs(output_path) + volumes = { - os.path.abspath(self.config.output): { + output_path: { "bind": output_path_guest, "mode": "rw" }, @@ -129,7 +133,13 @@ def search(self, client, output_path_guest, topic_path_guest, test_split_path_gu print("Evaluating results using trec_eval...") for file in os.listdir(self.config.output): - run = os.path.join(self.config.output, file) - print("###\n# {}\n###".format(run)) - subprocess.run("trec_eval/trec_eval {} {} {}".format(measures, self.config.qrels, run).split()) - print() + if not file.endswith("trec_eval"): + run = os.path.join(self.config.output, file) + print("###\n# {}\n###".format(run)) + try: + result = subprocess.check_output("trec_eval/trec_eval {} {} {}".format(measures, self.config.qrels, run).split()) + print(result.decode("UTF-8")) + with open("{}.trec_eval".format(run), "w+") as out: + out.write(result.decode("UTF-8")) + except subprocess.CalledProcessError: + print("Unable to evaluate {} - is it a run file?".format(run))