Skip to content

Commit

Permalink
Add BGE regressions with ONNX on BEIR (#2375)
Browse files Browse the repository at this point in the history 
+ maxlen issue with ONNX fixed in #2387
+ this PR simply copies scores from the non-ONNX versions, will circle back and readjust scores
  • Loading branch information
@lintool
lintool authored Feb 22, 2024
1 parent e6dcc67 commit 2ab4619
Show file tree
Hide file tree
Showing 177 changed files with 11,933 additions and 94 deletions.
67 changes: 34 additions & 33 deletions README.md
View file

Large diffs are not rendered by default.

66 changes: 64 additions & 2 deletions docs/regressions.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
<summary>BEIR (v1.0.0): BGE-base-en-v1.5</summary>

```bash
# Pre-encoded queries, original HNSW indexes
# Original HNSW indexes, pre-encoded queries
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw >& logs/log.beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw >& logs/log.beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw >& logs/log.beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw &
Expand Down Expand Up @@ -241,7 +241,38 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw >& logs/log.beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw >& logs/log.beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw &

# Pre-encoded queries, original HNSW indexes
# Original HNSW indexes, ONNX
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fiqa-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-fiqa-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gaming-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-gaming-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-mathematica-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-mathematica-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-onnx >& logs/log.beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-onnx &

# Quantized HNSW indexes, pre-encoded queries
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-int8 &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8 &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8 &
Expand Down Expand Up @@ -271,6 +302,37 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-int8 &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8 &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8 >& logs/log.beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8 &

# Quantized HNSW indexes, ONNX
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-trec-covid-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-bioasq-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-nfcorpus-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-nq-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-hotpotqa-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fiqa-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-fiqa-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-signal1m-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-trec-news-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-robust04-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-arguana-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-webis-touche2020-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-android-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-english-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gaming-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-gaming-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-gis-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-mathematica-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-mathematica-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-physics-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-programmers-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-stats-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-tex-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-unix-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-webmasters-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-cqadupstack-wordpress-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-quora-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-dbpedia-entity-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-scidocs-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-fever-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-climate-fever-bge-base-en-v1.5-hnsw-int8-onnx &
nohup python src/main/python/run_regression.py --index --verify --search --regression beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx >& logs/log.beir-v1.0.0-scifact-bge-base-en-v1.5-hnsw-int8-onnx &
```

</details>
Expand Down
Loading

0 comments on commit 2ab4619

Please sign in to comment.