Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: override existing results #1617

Merged
merged 9 commits into from
Dec 22, 2024
Merged

fix: override existing results #1617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
e51a04c
fix override existing results
Samoed Dec 20, 2024
e5c19de
lint
Samoed Dec 20, 2024
30e7443
fix tests
Samoed Dec 21, 2024
9634f5e
add tests with overwrite
Samoed Dec 21, 2024
15aa6a5
lint
Samoed Dec 21, 2024
182338b
update tests
Samoed Dec 21, 2024
e1e058a
lint
Samoed Dec 21, 2024
1ec1d1c
update
Samoed Dec 22, 2024
3941a04
lint
Samoed Dec 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 38 additions & 20 deletions mteb/evaluation/MTEB.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,35 +470,46 @@ def run(

existing_results = None
save_path = None
final_splits_to_run = task_eval_splits
missing_evaluations = self._get_missing_evaluations(
existing_results,
task_eval_splits,
task_subsets,
eval_subsets,
)
Comment on lines +474 to +479
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if existing_results=None here, then won't missing_evaluations contain all specified splits? Maybe this call can be omitted or simplified to an assignment?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If both task_eval_splits and task_subsets are None, all splits will be selected, but filtering for splits and subsets is still necessary. I think calling the same function is easier to understand than adding more complex logic.


if output_path:
save_path = output_path / f"{task.metadata.name}{task.save_suffix}.json"
if save_path.exists():
existing_results = TaskResult.from_disk(save_path)

if not overwrite_results:
# Unified call to get missing splits and subsets
missing_evaluations = self._get_missing_evaluations(
existing_results,
task_eval_splits,
task_subsets,
eval_subsets,
)

if overwrite_results:
final_splits_to_run = task_eval_splits
else:
# Determine final splits to run
final_splits_to_run = []
# We need to run any split that is fully missing or has missing subsets
for sp, info in missing_evaluations.items():
if info["whole_split_missing"] or info["missing_subsets"]:
final_splits_to_run.append(sp)

if not overwrite_results and len(final_splits_to_run) == 0:
logger.info(
f"{task.metadata.name} results already exists. Loading results from disk. Set overwrite_results=True to overwrite."
f"{task.metadata.name} results already exists. Loading results from disk."
f" Set overwrite_results=True to overwrite or `--overwrite`."
)
evaluation_results.append(existing_results)
del self.tasks[0] # empty memory
continue

# Unified call to get missing splits and subsets
missing_evaluations = self._get_missing_evaluations(
existing_results,
task_eval_splits,
task_subsets,
eval_subsets,
)

# Determine final splits to run
final_splits_to_run = []
# We need to run any split that is fully missing or has missing subsets
for sp, info in missing_evaluations.items():
if info["whole_split_missing"] or info["missing_subsets"]:
final_splits_to_run.append(sp)

# If no splits need to be run and results exist, skip
if not final_splits_to_run:
if existing_results is not None:
Expand All @@ -513,7 +524,7 @@ def run(

try:
task.check_if_dataset_is_superseded()
task.load_data(eval_splits=task_eval_splits, **kwargs)
task.load_data(**kwargs)
isaac-chung marked this conversation as resolved.
Show resolved Hide resolved

task_results = {}
evaluation_time = 0
Expand All @@ -527,8 +538,15 @@ def run(
# Determine subsets to run for this split
# If the whole split is missing, run all required subsets
# If only some subsets are missing, run only those
subsets_to_run = info["missing_subsets"]
if info["whole_split_missing"] and task_subsets is None:
subsets_to_run = (
info["missing_subsets"]
if not overwrite_results
else (eval_subsets or task_subsets)
)

if (
info["whole_split_missing"] or overwrite_results
) and task_subsets is None:
subsets_to_run = ["default"]

if co2_tracker:
Expand Down
81 changes: 69 additions & 12 deletions tests/test_evaluation/test_split_evaluation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ def test_one_missing_split(model, tasks, tmp_path):
eval_splits=["val", "test"],
output_folder=str(tmp_path / "testcase2"),
verbosity=2,
overwrite_results=True,
)

assert "MockRetrievalTask" == results2[0].task_name
Expand Down Expand Up @@ -93,12 +92,10 @@ def test_no_missing_splits(model, tasks, tmp_path):
eval_splits=["val", "test"],
output_folder=str(tmp_path / "testcase3"),
verbosity=2,
overwrite_results=True,
)

last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert "MockRetrievalTask" in last_evaluated_splits
assert len(last_evaluated_splits["MockRetrievalTask"]) == 0
assert len(last_evaluated_splits) == 0
assert results[0].scores.keys() == {"test", "val"}


Expand Down Expand Up @@ -144,7 +141,6 @@ def test_missing_language(model, multilingual_tasks, tmp_path):
output_folder=str(tmp_path / "missing_lang_test"),
verbosity=2,
eval_subsets=["eng", "fra"],
overwrite_results=True,
)

last_evaluated_splits = evaluation.get_last_evaluated_splits()
Expand Down Expand Up @@ -178,11 +174,9 @@ def test_no_missing_languages(model, multilingual_tasks, tmp_path):
output_folder=str(tmp_path / "no_missing_lang_test"),
verbosity=2,
eval_subsets=["eng", "fra"],
overwrite_results=True,
)
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert "MockMultilingualRetrievalTask" in last_evaluated_splits
assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 0
assert len(last_evaluated_splits) == 0
assert results[0].scores.keys() == {"test"}
assert len(results[0].scores["test"]) == 2
assert sorted(results[0].languages) == ["eng", "fra"]
Expand Down Expand Up @@ -210,7 +204,6 @@ def test_partial_languages(model, multilingual_tasks, tmp_path):
output_folder=str(tmp_path / "partial_lang_test"),
verbosity=2,
eval_subsets=["fra", "eng"],
overwrite_results=True,
)
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1
Expand Down Expand Up @@ -245,7 +238,6 @@ def test_multilingual_one_missing_split_no_missing_lang(
output_folder=str(tmp_path / "partial_langs_partial_splits"),
verbosity=2,
eval_subsets=["eng", "fra"],
overwrite_results=True,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should not affect how missing splits or langs are selected.

)

last_evaluated_splits = evaluation.get_last_evaluated_splits()
Expand Down Expand Up @@ -280,7 +272,6 @@ def test_multilingual_one_missing_lang_in_one_split(
output_folder=str(tmp_path / "one_lang_one_split"),
verbosity=2,
eval_subsets=["eng"],
overwrite_results=True,
)

last_evaluated_splits = evaluation.get_last_evaluated_splits()
Expand All @@ -296,7 +287,6 @@ def test_multilingual_one_missing_lang_in_one_split(
output_folder=str(tmp_path / "one_lang_one_split"),
verbosity=2,
eval_subsets=["eng", "fra"],
overwrite_results=True,
)

last_evaluated_splits = evaluation.get_last_evaluated_splits()
Expand All @@ -305,3 +295,70 @@ def test_multilingual_one_missing_lang_in_one_split(
# output merged result with previous results
assert results[0].scores.keys() == {"test", "val"}
assert len(results[0].scores["test"]) == 2


def test_all_splits_evaluated_with_overwrite(model, tasks, tmp_path):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does "all splits" mean here? There are test and val but only val is run here.

because all splits should be run with overwrite

Based on this comment, is it correct that "all" means all specified splits in the run method, and not all splits available in the task metadata's eval_splits?

Perhaps this can be spelled out, or preferably shown in additional test cases, where the 1st and 2nd runs have different eval_splits

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

means all specified splits in the run method,

Yes, all splits specified in evaluation.run

evaluation = MTEB(tasks=tasks)
results = evaluation.run(
model,
eval_splits=["val"],
output_folder=str(tmp_path / "all_splits_evaluated_with_overwrite"),
verbosity=2,
)

assert "MockRetrievalTask" == results[0].task_name
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert len(last_evaluated_splits["MockRetrievalTask"]) == 1
assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val"}
assert results[0].scores.keys() == {"val"}

results2 = evaluation.run(
model,
eval_splits=["val", "test"],
output_folder=str(tmp_path / "all_splits_evaluated_with_overwrite"),
verbosity=2,
overwrite_results=True,
)
assert "MockRetrievalTask" == results2[0].task_name
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert len(last_evaluated_splits["MockRetrievalTask"]) == 2
assert set(last_evaluated_splits["MockRetrievalTask"]) == {"val", "test"}
assert results2[0].scores.keys() == {"val", "test"}


def test_all_splits_subsets_evaluated_with_overwrite(
model, multilingual_tasks, tmp_path
):
evaluation = MTEB(tasks=multilingual_tasks)
results = evaluation.run(
model,
eval_splits=[
"test",
],
output_folder=str(tmp_path / "all_splits_subsets_evaluated_with_overwrite"),
verbosity=2,
eval_subsets=["fra"],
)
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert "MockMultilingualRetrievalTask" in last_evaluated_splits
assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1
assert results[0].scores.keys() == {"test"}
for split in ["test"]:
assert len(results[0].scores[split]) == 1
assert sorted(results[0].languages) == ["fra"]

results2 = evaluation.run(
model,
eval_splits=["test"],
output_folder=str(tmp_path / "all_splits_subsets_evaluated_with_overwrite"),
verbosity=2,
eval_subsets=["fra", "eng"],
overwrite_results=True,
)
last_evaluated_splits = evaluation.get_last_evaluated_splits()
assert "MockMultilingualRetrievalTask" in last_evaluated_splits
assert len(last_evaluated_splits["MockMultilingualRetrievalTask"]) == 1
assert results2[0].scores.keys() == {"test"}
for split in ["test"]:
assert len(results2[0].scores[split]) == 2
assert sorted(results2[0].languages) == ["eng", "fra"]
Loading