From ca5a2beb7164013e787e0124c7d79b5c751a2d60 Mon Sep 17 00:00:00 2001 From: Jack Lin Date: Fri, 21 Apr 2023 19:52:43 +0800 Subject: [PATCH] update 2cr report (#1507) --- docs/2cr/msmarco-v1-passage.html | 212 +++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) diff --git a/docs/2cr/msmarco-v1-passage.html b/docs/2cr/msmarco-v1-passage.html index eb64bdd06..b513db5d5 100644 --- a/docs/2cr/msmarco-v1-passage.html +++ b/docs/2cr/msmarco-v1-passage.html @@ -2885,6 +2885,214 @@

MS MARCO V1 Passage

+ + + + + +[7] +Aggretriever-DistilBERT: on-the-fly query inference +0.4301 +0.6816 +0.8023 + +0.4329 +0.6726 +0.8351 + +0.3412 +0.9604 + + + + +
+ + + + + + +
+
+Command to generate run on TREC 2019 queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-distilbert \
+  --topics dl19-passage \
+  --encoder castorini/aggretriever-distilbert \
+  --output run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl19.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -l 2 -m map dl19-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl19.txt
+python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl19-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl19.txt
+python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl19-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl19.txt
+
+
+ +
+
+ Command to generate run on TREC 2020 queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-distilbert \
+  --topics dl20 \
+  --encoder castorini/aggretriever-distilbert \
+  --output run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl20.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl20.txt
+python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl20.txt
+python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dl20.txt
+
+
+ +
+
+ Command to generate run on dev queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-distilbert \
+  --topics msmarco-passage-dev-subset \
+  --encoder castorini/aggretriever-distilbert \
+  --output run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dev.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dev.txt
+python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset run.msmarco-v1-passage.Aggretriever-Distilbert-otf.dev.txt
+
+
+ +
+
+ + +
+ + + + +[7] +Aggretriever-coCondenser: on-the-fly query inference +0.4350 +0.6837 +0.8078 + +0.4710 +0.6972 +0.8555 + +0.3619 +0.9735 + + + + +
+ + + + + + +
+
+Command to generate run on TREC 2019 queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-cocondenser \
+  --topics dl19-passage \
+  --encoder castorini/aggretriever-cocondenser \
+  --output run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl19.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -l 2 -m map dl19-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl19.txt
+python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl19-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl19.txt
+python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl19-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl19.txt
+
+
+ +
+
+ Command to generate run on TREC 2020 queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-cocondenser \
+  --topics dl20 \
+  --encoder castorini/aggretriever-cocondenser \
+  --output run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl20.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl20.txt
+python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl20.txt
+python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dl20.txt
+
+
+ +
+
+ Command to generate run on dev queries: + +
+
python -m pyserini.search.faiss \
+  --threads 16 --batch-size 512 \
+  --index msmarco-passage.aggretriever-cocondenser \
+  --topics msmarco-passage-dev-subset \
+  --encoder castorini/aggretriever-cocondenser \
+  --output run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dev.txt
+
+Evaluation commands: + +
+
python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dev.txt
+python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset run.msmarco-v1-passage.Aggretriever-coCondenser-otf.dev.txt
+
+
+ +
+
+ +
@@ -2920,6 +3128,10 @@

MS MARCO V1 Passage

SLIM: Sparsified Late Interaction for Multi-Vector Retrieval with Inverted Indexes. arXiv:2302.06587, Feburary 2023.

+

  • [7] Sheng-Chieh Lin, Minghan Li and Jimmy Lin. +Aggretriever: A Simple Approach to Aggregate Textual Representation for Robust Dense Passage Retrieval. +arXiv:2208.00511, July 2022.

    • +