Tweak score tolerance for regressions (#2545)

castorini · Jul 17, 2024 · bb55c0b · bb55c0b
1 parent bfb9394
commit bb55c0b
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/src/main/python/run_regression.py b/src/main/python/run_regression.py
@@ -234,6 +234,7 @@ def construct_convert_commands(yaml_data):
 
 beir_hnsw_onnx = defaultdict(lambda: 0.003)
 beir_hnsw_onnx['ArguAna'] = 0.02
+beir_hnsw_onnx['BioASQ'] = 0.005
 beir_hnsw_onnx['CQADupStack-wordpress'] = 0.004
 beir_hnsw_onnx['DBPedia'] = 0.006
 beir_hnsw_onnx['FEVER'] = 0.007
@@ -244,6 +245,7 @@ def construct_convert_commands(yaml_data):
 beir_hnsw_onnx['TREC-NEWS'] = 0.02
 
 beir_hnsw_cached = defaultdict(lambda: 0.003)
+beir_hnsw_cached['BioASQ'] = 0.005
 beir_hnsw_cached['DBPedia'] = 0.006
 beir_hnsw_cached['FEVER'] = 0.008
 beir_hnsw_cached['FiQA-2018'] = 0.008
@@ -294,6 +296,7 @@ def construct_convert_commands(yaml_data):
 
 dl20_flat_int8_onnx = defaultdict(lambda: 0.002)
 dl20_flat_int8_onnx['bge-flat-int8-onnx'] = 0.004
+dl20_flat_int8_onnx['cos-dpr-distil-flat-int8-onnx'] = 0.004
 dl20_flat_int8_cached = defaultdict(lambda: 0.002)
 dl20_flat_int8_cached['bge-flat-int8-cached'] = 0.005
 dl20_flat_int8_cached['cos-dpr-distil-flat-int8-cached'] = 0.004
@@ -328,7 +331,7 @@ def construct_convert_commands(yaml_data):
 dl19_hnsw_int8_onnx['cos-dpr-distil-hnsw-int8-onnx'] = 0.025
 dl19_hnsw_int8_cached = defaultdict(lambda: 0.01)
 dl19_hnsw_int8_cached['bge-hnsw-int8-cached'] = 0.015
-dl19_hnsw_int8_cached['cohere-embed-english-v3.0-hnsw-int8-cached'] = 0.015
+dl19_hnsw_int8_cached['cohere-embed-english-v3.0-hnsw-int8-cached'] = 0.02
 dl19_hnsw_int8_cached['cos-dpr-distil-hnsw-int8-cached'] = 0.025
 dl19_hnsw_int8_cached['openai-ada2-hnsw-int8-cached'] = 0.015
 dl19_hnsw_onnx = defaultdict(lambda: 0.015)
@@ -347,7 +350,7 @@ def construct_convert_commands(yaml_data):
 dl20_hnsw_int8_onnx = defaultdict(lambda: 0.01)
 dl20_hnsw_int8_cached = defaultdict(lambda: 0.01)
 dl20_hnsw_int8_cached['bge-hnsw-int8-cached'] = 0.015
-dl20_hnsw_int8_cached['cohere-embed-english-v3.0-hnsw-int8-cached'] = 0.012
+dl20_hnsw_int8_cached['cohere-embed-english-v3.0-hnsw-int8-cached'] = 0.015
 dl20_hnsw_onnx = defaultdict(lambda: 0.015)
 dl20_hnsw_cached = defaultdict(lambda: 0.01)
 dl20_hnsw_cached['bge-hnsw-cached'] = 0.015