From f0b37dd28ffec543a9ef107a52297b30199b69f1 Mon Sep 17 00:00:00 2001 From: Jimmy Lin Date: Thu, 22 Feb 2024 16:50:49 -0500 Subject: [PATCH] Update BEIR scores using BGE w/ ONNX (#2388) + Tweaked scores by averaging over 4 trials + Added documentation for Cohere DL19/20 --- docs/regressions.md | 4 ++++ ...-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...sions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.md | 6 +++--- ...s-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ...ssions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.md | 6 +++--- ...v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.md | 4 ++-- ...-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.md | 6 +++--- ....0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...ir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...dupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ....0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ....0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...ir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...adupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ....0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...qadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...eir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.md | 4 ++-- ...essions-beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ...ions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ...sions-beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...ns-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...essions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ...ions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.md | 4 ++-- ...-beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.md | 4 ++-- ...-beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.md | 2 +- ...beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...ions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.md | 6 +++--- ...ns-beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.md | 2 +- ...eir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...ons-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.md | 6 +++--- ...0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.md | 6 +++--- ...r-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.md | 2 +- .../beir/extract_avg_hnsw_regression_scores_from_log.py | 7 ++++--- src/main/python/regressions-batch03.txt | 4 ++++ ...eir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- .../beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml | 6 +++--- ...beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- .../beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml | 6 +++--- ....0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...ir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml | 4 ++-- ...qadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...qadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml | 6 +++--- ....0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...qadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...pstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- ...-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- ...1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ....0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...upstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...dupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- ...r-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.yaml | 4 ++-- .../beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...ir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- .../beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...ir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- .../beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- .../beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- .../beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...ir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- .../beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml | 4 ++-- ...eir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 4 ++-- ...eir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 2 +- ...ir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- .../beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml | 6 +++--- .../beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- ...r-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- .../beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml | 6 +++--- ...0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml | 6 +++--- ...v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.yaml | 2 +- 89 files changed, 160 insertions(+), 151 deletions(-) diff --git a/docs/regressions.md b/docs/regressions.md index fe91f18135..6606cb4fb8 100644 --- a/docs/regressions.md +++ b/docs/regressions.md @@ -95,6 +95,8 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-bge-base-en-v1.5-hnsw >& logs/log.dl19-passage-bge-base-en-v1.5-hnsw & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-bge-base-en-v1.5-hnsw-int8 >& logs/log.dl19-passage-bge-base-en-v1.5-hnsw-int8 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-openai-ada2 >& logs/log.dl19-passage-openai-ada2 & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-cohere-embed-english-v3-hnsw >& logs/log.dl19-passage-cohere-embed-english-v3-hnsw & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-cohere-embed-english-v3-hnsw-int8 >& logs/log.dl19-passage-cohere-embed-english-v3-hnsw-int8 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-splade-pp-ed-onnx >& logs/log.dl19-passage-splade-pp-ed-onnx & nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-splade-pp-sd-onnx >& logs/log.dl19-passage-splade-pp-sd-onnx & @@ -132,6 +134,8 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-bge-base-en-v1.5-hnsw >& logs/log.dl20-passage-bge-base-en-v1.5-hnsw & nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-bge-base-en-v1.5-hnsw-int8 >& logs/log.dl20-passage-bge-base-en-v1.5-hnsw-int8 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-openai-ada2 >& logs/log.dl20-passage-openai-ada2 & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-cohere-embed-english-v3-hnsw >& logs/log.dl20-passage-cohere-embed-english-v3-hnsw & +nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-cohere-embed-english-v3-hnsw-int8 >& logs/log.dl20-passage-cohere-embed-english-v3-hnsw-int8 & nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-splade-pp-ed-onnx >& logs/log.dl20-passage-splade-pp-ed-onnx & nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-splade-pp-sd-onnx >& logs/log.dl20-passage-splade-pp-sd-onnx & diff --git a/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.md index 264b25ac2c..07c47717d0 100644 --- a/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): ArguAna | 0.635 | +| BEIR (v1.0.0): ArguAna | 0.621 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): ArguAna | 0.991 | +| BEIR (v1.0.0): ArguAna | 0.971 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): ArguAna | 0.996 | +| BEIR (v1.0.0): ArguAna | 0.994 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.md index 7e5cdbd13d..ca7a6cf30c 100644 --- a/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): ArguAna | 0.636 | +| BEIR (v1.0.0): ArguAna | 0.623 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): ArguAna | 0.992 | +| BEIR (v1.0.0): ArguAna | 0.972 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): ArguAna | 0.996 | +| BEIR (v1.0.0): ArguAna | 0.993 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.md index ac58c0b701..137ebcc815 100644 --- a/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): BioASQ | 0.407 | +| BEIR (v1.0.0): BioASQ | 0.408 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): BioASQ | 0.624 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): BioASQ | 0.795 | +| BEIR (v1.0.0): BioASQ | 0.797 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.md index 403ae522f3..13a3499638 100644 --- a/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): BioASQ | 0.410 | +| BEIR (v1.0.0): BioASQ | 0.414 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): BioASQ | 0.622 | +| BEIR (v1.0.0): BioASQ | 0.628 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): BioASQ | 0.794 | +| BEIR (v1.0.0): BioASQ | 0.802 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.md index a2a7beffd5..591e1e4880 100644 --- a/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Climate-FEVER | 0.309 | +| BEIR (v1.0.0): Climate-FEVER | 0.308 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Climate-FEVER | 0.633 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.md index 19260d7a1a..32ec12fce7 100644 --- a/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.md @@ -76,9 +76,9 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): Climate-FEVER | 0.312 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Climate-FEVER | 0.636 | +| BEIR (v1.0.0): Climate-FEVER | 0.635 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Climate-FEVER | 0.829 | +| BEIR (v1.0.0): Climate-FEVER | 0.830 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.md index 78fa3d5e2e..a3876c875b 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): CQADupStack-android | 0.509 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-android | 0.844 | +| BEIR (v1.0.0): CQADupStack-android | 0.843 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-android | 0.962 | diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.md index a966c3b0c2..70dfc34dd1 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-android | 0.507 | +| BEIR (v1.0.0): CQADupStack-android | 0.508 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-android | 0.845 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.md index 67474085a1..0d620d673b 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -78,7 +78,7 @@ With the above commands, you should be able to reproduce the following results: | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-english | 0.756 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-english | 0.883 | +| BEIR (v1.0.0): CQADupStack-english | 0.882 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.md index 8709115f69..512697ff4d 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-english | 0.485 | +| BEIR (v1.0.0): CQADupStack-english | 0.484 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-english | 0.757 | +| BEIR (v1.0.0): CQADupStack-english | 0.756 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-english | 0.882 | +| BEIR (v1.0.0): CQADupStack-english | 0.881 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.md index 688b66362a..12aab8bfd7 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-gis | 0.415 | +| BEIR (v1.0.0): CQADupStack-gis | 0.416 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-gis | 0.767 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.md index f9a20b74fa..8d4a222837 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-gis | 0.412 | +| BEIR (v1.0.0): CQADupStack-gis | 0.413 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-gis | 0.767 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.md index 9484b7be44..5d245c8b8e 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-physics | 0.474 | +| BEIR (v1.0.0): CQADupStack-physics | 0.473 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-physics | 0.810 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.md index 444f058cf6..d208972cd5 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,9 +74,9 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-programmers | 0.425 | +| BEIR (v1.0.0): CQADupStack-programmers | 0.424 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-programmers | 0.786 | +| BEIR (v1.0.0): CQADupStack-programmers | 0.787 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-programmers | 0.934 | diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.md index ad1def1618..d4ed071319 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-stats | 0.371 | +| BEIR (v1.0.0): CQADupStack-stats | 0.370 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-stats | 0.672 | +| BEIR (v1.0.0): CQADupStack-stats | 0.671 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-stats | 0.852 | +| BEIR (v1.0.0): CQADupStack-stats | 0.854 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.md index 4cea46769c..2a93931513 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): CQADupStack-stats | 0.373 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-stats | 0.673 | +| BEIR (v1.0.0): CQADupStack-stats | 0.672 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-stats | 0.849 | diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.md index af795bbfd1..2ad893ce1b 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -78,7 +78,7 @@ With the above commands, you should be able to reproduce the following results: | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-tex | 0.648 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-tex | 0.852 | +| BEIR (v1.0.0): CQADupStack-tex | 0.853 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.md index b2f55d0e9b..8beca6d450 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.md @@ -78,7 +78,7 @@ With the above commands, you should be able to reproduce the following results: | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-tex | 0.647 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-tex | 0.852 | +| BEIR (v1.0.0): CQADupStack-tex | 0.853 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.md index 6547c4f66e..60da2904c4 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-unix | 0.420 | +| BEIR (v1.0.0): CQADupStack-unix | 0.421 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-unix | 0.777 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.md index 24b95106d3..91de6cac01 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-webmasters | 0.411 | +| BEIR (v1.0.0): CQADupStack-webmasters | 0.409 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-webmasters | 0.780 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.md index e366b0c62f..eb039c94c6 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): CQADupStack-webmasters | 0.406 | +| BEIR (v1.0.0): CQADupStack-webmasters | 0.407 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-webmasters | 0.777 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.md index 6a53d75df3..4fc6dfc803 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): CQADupStack-wordpress | 0.354 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-wordpress | 0.706 | +| BEIR (v1.0.0): CQADupStack-wordpress | 0.707 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-wordpress | 0.886 | diff --git a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.md index af798f65ce..1a7bd158e2 100644 --- a/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): CQADupStack-wordpress | 0.355 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): CQADupStack-wordpress | 0.705 | +| BEIR (v1.0.0): CQADupStack-wordpress | 0.703 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): CQADupStack-wordpress | 0.886 | diff --git a/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.md index 1c62b96db6..63a2e6484a 100644 --- a/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): DBPedia | 0.407 | +| BEIR (v1.0.0): DBPedia | 0.408 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): DBPedia | 0.527 | +| BEIR (v1.0.0): DBPedia | 0.528 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): DBPedia | 0.776 | +| BEIR (v1.0.0): DBPedia | 0.778 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.md index 7b814ee025..665d1d95f5 100644 --- a/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.md @@ -74,9 +74,9 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): DBPedia | 0.407 | +| BEIR (v1.0.0): DBPedia | 0.408 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): DBPedia | 0.528 | +| BEIR (v1.0.0): DBPedia | 0.529 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): DBPedia | 0.778 | diff --git a/docs/regressions/regressions-beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.md index 36ec96e682..f4a3f105cd 100644 --- a/docs/regressions/regressions-beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): FEVER | 0.861 | +| BEIR (v1.0.0): FEVER | 0.860 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): FEVER | 0.967 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.md index b874344cb3..ad7845c8f4 100644 --- a/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -76,9 +76,9 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): HotpotQA | 0.722 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): HotpotQA | 0.866 | +| BEIR (v1.0.0): HotpotQA | 0.867 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): HotpotQA | 0.935 | +| BEIR (v1.0.0): HotpotQA | 0.936 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.md index d7c23f9470..9a02c782c5 100644 --- a/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.md @@ -78,7 +78,7 @@ With the above commands, you should be able to reproduce the following results: | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): HotpotQA | 0.866 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): HotpotQA | 0.936 | +| BEIR (v1.0.0): HotpotQA | 0.935 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.md index 194f068422..5d7e1235ee 100644 --- a/docs/regressions/regressions-beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,9 +74,9 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): NFCorpus | 0.373 | +| BEIR (v1.0.0): NFCorpus | 0.374 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): NFCorpus | 0.338 | +| BEIR (v1.0.0): NFCorpus | 0.339 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): NFCorpus | 0.657 | diff --git a/docs/regressions/regressions-beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.md index 5e41529e05..426b122208 100644 --- a/docs/regressions/regressions-beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): NQ | 0.538 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): NQ | 0.940 | +| BEIR (v1.0.0): NQ | 0.939 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): NQ | 0.984 | diff --git a/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.md index 64210651a8..4724a91caf 100644 --- a/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Quora | 0.888 | +| BEIR (v1.0.0): Quora | 0.887 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Quora | 0.997 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.md index e9b0d9c6cb..7ac09dbfcb 100644 --- a/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Quora | 0.889 | +| BEIR (v1.0.0): Quora | 0.888 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Quora | 0.997 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.md index 223982da8d..02633a2a64 100644 --- a/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Robust04 | 0.447 | +| BEIR (v1.0.0): Robust04 | 0.445 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Robust04 | 0.347 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Robust04 | 0.596 | +| BEIR (v1.0.0): Robust04 | 0.592 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.md index e6da2418b8..f41b43eea3 100644 --- a/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Robust04 | 0.447 | +| BEIR (v1.0.0): Robust04 | 0.444 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Robust04 | 0.350 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Robust04 | 0.596 | +| BEIR (v1.0.0): Robust04 | 0.595 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.md index 181cfd4ece..6947f72511 100644 --- a/docs/regressions/regressions-beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -76,9 +76,9 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): SCIDOCS | 0.217 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): SCIDOCS | 0.494 | +| BEIR (v1.0.0): SCIDOCS | 0.493 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): SCIDOCS | 0.785 | +| BEIR (v1.0.0): SCIDOCS | 0.784 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.md index 7b6b13efaa..5078b8f341 100644 --- a/docs/regressions/regressions-beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,7 +74,7 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): SciFact | 0.741 | +| BEIR (v1.0.0): SciFact | 0.740 | | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): SciFact | 0.969 | | **R@1000** | **BGE-base-en-v1.5**| diff --git a/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.md index 7d05d4efe2..75686c16ea 100644 --- a/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Signal-1M | 0.278 | +| BEIR (v1.0.0): Signal-1M | 0.277 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Signal-1M | 0.291 | +| BEIR (v1.0.0): Signal-1M | 0.292 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Signal-1M | 0.490 | +| BEIR (v1.0.0): Signal-1M | 0.498 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.md index d987e227e8..5d32bc4665 100644 --- a/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Signal-1M | 0.282 | +| BEIR (v1.0.0): Signal-1M | 0.280 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Signal-1M | 0.298 | +| BEIR (v1.0.0): Signal-1M | 0.294 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Signal-1M | 0.500 | +| BEIR (v1.0.0): Signal-1M | 0.495 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.md index c80f94e482..78a9924fe5 100644 --- a/docs/regressions/regressions-beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.md @@ -78,7 +78,7 @@ With the above commands, you should be able to reproduce the following results: | **R@100** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): TREC-COVID | 0.141 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): TREC-COVID | 0.477 | +| BEIR (v1.0.0): TREC-COVID | 0.476 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.md index 422710a48c..3aa35d4a74 100644 --- a/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): TREC-NEWS | 0.432 | +| BEIR (v1.0.0): TREC-NEWS | 0.434 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): TREC-NEWS | 0.490 | +| BEIR (v1.0.0): TREC-NEWS | 0.488 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): TREC-NEWS | 0.777 | +| BEIR (v1.0.0): TREC-NEWS | 0.774 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.md index 8529ac99ef..d3f0300aa7 100644 --- a/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): TREC-NEWS | 0.441 | +| BEIR (v1.0.0): TREC-NEWS | 0.439 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): TREC-NEWS | 0.488 | +| BEIR (v1.0.0): TREC-NEWS | 0.492 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): TREC-NEWS | 0.770 | +| BEIR (v1.0.0): TREC-NEWS | 0.780 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.md b/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.md index 9c367ef466..29532e5e3e 100644 --- a/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.md @@ -74,11 +74,11 @@ With the above commands, you should be able to reproduce the following results: | **nDCG@10** | **BGE-base-en-v1.5**| |:-------------------------------------------------------------------------------------------------------------|-----------| -| BEIR (v1.0.0): Webis-Touche2020 | 0.252 | +| BEIR (v1.0.0): Webis-Touche2020 | 0.251 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Webis-Touche2020 | 0.488 | +| BEIR (v1.0.0): Webis-Touche2020 | 0.487 | | **R@1000** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Webis-Touche2020 | 0.831 | +| BEIR (v1.0.0): Webis-Touche2020 | 0.833 | Note that due to the non-deterministic nature of HNSW indexing, results may differ slightly between each experimental run. Nevertheless, scores are generally within 0.005 of the reference values recorded in [our YAML configuration file](../../src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml). diff --git a/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.md b/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.md index 86c292c7d4..c1a4eb2a02 100644 --- a/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.md +++ b/docs/regressions/regressions-beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.md @@ -76,7 +76,7 @@ With the above commands, you should be able to reproduce the following results: |:-------------------------------------------------------------------------------------------------------------|-----------| | BEIR (v1.0.0): Webis-Touche2020 | 0.257 | | **R@100** | **BGE-base-en-v1.5**| -| BEIR (v1.0.0): Webis-Touche2020 | 0.486 | +| BEIR (v1.0.0): Webis-Touche2020 | 0.487 | | **R@1000** | **BGE-base-en-v1.5**| | BEIR (v1.0.0): Webis-Touche2020 | 0.831 | diff --git a/src/main/python/beir/extract_avg_hnsw_regression_scores_from_log.py b/src/main/python/beir/extract_avg_hnsw_regression_scores_from_log.py index 9c9621469d..3fef97917f 100644 --- a/src/main/python/beir/extract_avg_hnsw_regression_scores_from_log.py +++ b/src/main/python/beir/extract_avg_hnsw_regression_scores_from_log.py @@ -53,7 +53,7 @@ for key in sorted(beir_keys): print(key) for metric in ['nDCG@10', 'R@100', 'R@1000']: - command = f'tail -n 5 logs/log.beir-v1.0.0-{key}-bge-base-en-v1.5-hnsw-int8_* | grep "{metric}\s"' + command = f'tail -n 5 logs/log.beir-v1.0.0-{key}-bge-base-en-v1.5-hnsw-int8-onnx_* | grep "{metric}\s"' p = subprocess.run(command, shell=True, text=True, capture_output=True) output = p.stdout scores = [] @@ -62,5 +62,6 @@ match = re.search(pattern, line) if match: scores.append(float(match.group(1))) - avg = round(sum(scores)/len(scores) * 10 ** 3) / (10 ** 3) - print(f' {metric}:\n - {avg:.3f}') + if len(scores) > 0: + avg = round(sum(scores)/len(scores) * 10 ** 3) / (10 ** 3) + print(f' {metric} (avg over {len(scores)}):\n - {avg:.3f}') diff --git a/src/main/python/regressions-batch03.txt b/src/main/python/regressions-batch03.txt index 1911edde98..7b8ed54344 100644 --- a/src/main/python/regressions-batch03.txt +++ b/src/main/python/regressions-batch03.txt @@ -148,6 +148,8 @@ python src/main/python/run_regression.py --search --regression dl19-passage-cos- python src/main/python/run_regression.py --search --regression dl19-passage-cos-dpr-distil-fw > logs/log.dl19-passage-cos-dpr-distil-fw 2>&1 python src/main/python/run_regression.py --search --regression dl19-passage-cos-dpr-distil-lexlsh > logs/log.dl19-passage-cos-dpr-distil-lexlsh 2>&1 python src/main/python/run_regression.py --search --regression dl19-passage-openai-ada2 > logs/log.dl19-passage-openai-ada2 2>&1 +python src/main/python/run_regression.py --search --regression dl19-passage-cohere-embed-english-v3-hnsw > logs/log.dl19-passage-cohere-embed-english-v3-hnsw 2>&1 +python src/main/python/run_regression.py --search --regression dl19-passage-cohere-embed-english-v3-hnsw-int8 > logs/log.dl19-passage-cohere-embed-english-v3-hnsw-int8 2>&1 python src/main/python/run_regression.py --search --regression dl19-passage-unicoil > logs/log.dl19-passage-unicoil 2>&1 python src/main/python/run_regression.py --search --regression dl19-passage-unicoil-noexp > logs/log.dl19-passage-unicoil-noexp 2>&1 @@ -184,6 +186,8 @@ python src/main/python/run_regression.py --search --regression dl20-passage-cos- python src/main/python/run_regression.py --search --regression dl20-passage-cos-dpr-distil-fw > logs/log.dl20-passage-cos-dpr-distil-fw 2>&1 python src/main/python/run_regression.py --search --regression dl20-passage-cos-dpr-distil-lexlsh > logs/log.dl20-passage-cos-dpr-distil-lexlsh 2>&1 python src/main/python/run_regression.py --search --regression dl20-passage-openai-ada2 > logs/log.dl20-passage-openai-ada2 2>&1 +python src/main/python/run_regression.py --search --regression dl20-passage-cohere-embed-english-v3-hnsw > logs/log.dl20-passage-cohere-embed-english-v3-hnsw 2>&1 +python src/main/python/run_regression.py --search --regression dl20-passage-cohere-embed-english-v3-hnsw-int8 > logs/log.dl20-passage-cohere-embed-english-v3-hnsw-int8 2>&1 python src/main/python/run_regression.py --search --regression dl20-passage-unicoil > logs/log.dl20-passage-unicoil 2>&1 python src/main/python/run_regression.py --search --regression dl20-passage-unicoil-noexp > logs/log.dl20-passage-unicoil-noexp 2>&1 diff --git a/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 35ab16c069..64be2ddc55 100644 --- a/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.635 + - 0.621 R@100: - - 0.991 + - 0.971 R@1000: - - 0.996 + - 0.994 diff --git a/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml index d652e49331..bb92761f78 100644 --- a/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.636 + - 0.623 R@100: - - 0.992 + - 0.972 R@1000: - - 0.996 + - 0.993 diff --git a/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 1d7e70a90c..15f3b1939a 100644 --- a/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 5000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.407 + - 0.408 R@100: - 0.624 R@1000: - - 0.795 + - 0.797 diff --git a/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml index d2bccc0e75..0766481eeb 100644 --- a/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 5000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.410 + - 0.414 R@100: - - 0.622 + - 0.628 R@1000: - - 0.794 + - 0.802 diff --git a/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.yaml index c5389df04a..a92e1f4267 100644 --- a/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.309 + - 0.308 R@100: - 0.633 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml index 6bbc4b0456..cf484405d2 100644 --- a/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.312 R@100: - - 0.636 + - 0.635 R@1000: - - 0.829 + - 0.830 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.yaml index f070b27fcc..747a69e774 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.509 R@100: - - 0.844 + - 0.843 R@1000: - 0.962 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.yaml index fdc7dfeb73..6dd0c0cb11 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.507 + - 0.508 R@100: - 0.845 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 17d77456fd..852282c4c5 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -50,4 +50,4 @@ models: R@100: - 0.756 R@1000: - - 0.883 + - 0.882 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml index 4fecc69483..6d1a6188bb 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.485 + - 0.484 R@100: - - 0.757 + - 0.756 R@1000: - - 0.882 + - 0.881 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 59b0bd553e..907637da57 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.415 + - 0.416 R@100: - 0.767 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.yaml index 46bd18a338..a141080216 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.412 + - 0.413 R@100: - 0.767 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.yaml index c8435ab18b..cac500da59 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.474 + - 0.473 R@100: - 0.810 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.yaml index cd8fa14979..731ed302a4 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.425 + - 0.424 R@100: - - 0.786 + - 0.787 R@1000: - 0.934 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml index ace76d4efa..370620af96 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.371 + - 0.370 R@100: - - 0.672 + - 0.671 R@1000: - - 0.852 + - 0.854 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.yaml index 121008f2ad..e63d851b16 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.373 R@100: - - 0.673 + - 0.672 R@1000: - 0.849 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 1fe2bf7086..7c78daeef5 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -50,4 +50,4 @@ models: R@100: - 0.648 R@1000: - - 0.852 + - 0.853 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml index 2b03170058..17b0a36438 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx.yaml @@ -50,4 +50,4 @@ models: R@100: - 0.647 R@1000: - - 0.852 + - 0.853 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.yaml index b25345ea41..33932820f5 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.420 + - 0.421 R@100: - 0.777 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.yaml index e710045121..05d71f4536 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.411 + - 0.409 R@100: - 0.780 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.yaml index fddc959550..0894172f91 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.406 + - 0.407 R@100: - 0.777 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 574e4c3109..7313ac982e 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.354 R@100: - - 0.706 + - 0.707 R@1000: - 0.886 diff --git a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.yaml index ef619cfe9e..17c183216c 100644 --- a/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.355 R@100: - - 0.705 + - 0.703 R@1000: - 0.886 diff --git a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 42ce1d1a39..fd8c70db42 100644 --- a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.407 + - 0.408 R@100: - - 0.527 + - 0.528 R@1000: - - 0.776 + - 0.778 diff --git a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.yaml index 09b2a96072..fab1540db3 100644 --- a/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.407 + - 0.408 R@100: - - 0.528 + - 0.529 R@1000: - 0.778 diff --git a/src/main/resources/regression/beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.yaml index ab0d957c4f..9ba43d6eac 100644 --- a/src/main/resources/regression/beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.861 + - 0.860 R@100: - 0.967 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 3f69eee8c6..c6a8cdc112 100644 --- a/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.722 R@100: - - 0.866 + - 0.867 R@1000: - - 0.935 + - 0.936 diff --git a/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml index a3eb47a5fc..de46999b33 100644 --- a/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx.yaml @@ -50,4 +50,4 @@ models: R@100: - 0.866 R@1000: - - 0.936 + - 0.935 diff --git a/src/main/resources/regression/beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 5e9997890e..88bef4139b 100644 --- a/src/main/resources/regression/beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.373 + - 0.374 R@100: - - 0.338 + - 0.339 R@1000: - 0.657 diff --git a/src/main/resources/regression/beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 314131cf58..b1f34879a2 100644 --- a/src/main/resources/regression/beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.538 R@100: - - 0.940 + - 0.939 R@1000: - 0.984 diff --git a/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 15abd8846a..988bb3c930 100644 --- a/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.888 + - 0.887 R@100: - 0.997 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.yaml index 1fb3ada4e3..192eb0761f 100644 --- a/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.889 + - 0.888 R@100: - 0.997 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 0c471b029c..6c8cd4969c 100644 --- a/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.447 + - 0.445 R@100: - 0.347 R@1000: - - 0.596 + - 0.592 diff --git a/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml index e063d48983..9c0087b609 100644 --- a/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.447 + - 0.444 R@100: - 0.350 R@1000: - - 0.596 + - 0.595 diff --git a/src/main/resources/regression/beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 271ab290fc..3492f86e40 100644 --- a/src/main/resources/regression/beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.217 R@100: - - 0.494 + - 0.493 R@1000: - - 0.785 + - 0.784 diff --git a/src/main/resources/regression/beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 6fafc684b5..31e310f2ce 100644 --- a/src/main/resources/regression/beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,7 +46,7 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.741 + - 0.740 R@100: - 0.969 R@1000: diff --git a/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml index 0c777ad726..a91c1709d4 100644 --- a/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.278 + - 0.277 R@100: - - 0.291 + - 0.292 R@1000: - - 0.490 + - 0.498 diff --git a/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml index eec4d9d2dd..2ee7f9b616 100644 --- a/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.282 + - 0.280 R@100: - - 0.298 + - 0.294 R@1000: - - 0.500 + - 0.495 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml index ad02ef5c16..3eb159c2e5 100644 --- a/src/main/resources/regression/beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx.yaml @@ -50,4 +50,4 @@ models: R@100: - 0.141 R@1000: - - 0.477 + - 0.476 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml index a06bdbdc50..e92bcfead0 100644 --- a/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.432 + - 0.434 R@100: - - 0.490 + - 0.488 R@1000: - - 0.777 + - 0.774 diff --git a/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml index ddace9670b..604849c180 100644 --- a/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.441 + - 0.439 R@100: - - 0.488 + - 0.492 R@1000: - - 0.770 + - 0.780 diff --git a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml index ca3f94cbfe..6d8326f651 100644 --- a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx.yaml @@ -46,8 +46,8 @@ models: params: -generator VectorQueryGenerator -topicField title -removeQuery -threads 16 -hits 1000 -efSearch 1000 -encoder BgeBaseEn15 results: nDCG@10: - - 0.252 + - 0.251 R@100: - - 0.488 + - 0.487 R@1000: - - 0.831 + - 0.833 diff --git a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.yaml b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.yaml index 59b3582efd..dc9cfed35c 100644 --- a/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.yaml +++ b/src/main/resources/regression/beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx.yaml @@ -48,6 +48,6 @@ models: nDCG@10: - 0.257 R@100: - - 0.486 + - 0.487 R@1000: - 0.831